2024-12-05 05:16:05,082 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 05:16:05,092 main DEBUG Took 0.008309 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 05:16:05,092 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 05:16:05,092 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 05:16:05,093 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 05:16:05,094 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,100 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 05:16:05,111 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,113 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,113 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,114 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,114 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,115 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,115 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,116 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,116 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,117 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,117 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,117 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,118 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,118 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,118 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,119 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,119 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,120 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,120 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,120 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,121 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 05:16:05,121 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,121 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 05:16:05,122 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 05:16:05,124 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 05:16:05,125 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 05:16:05,126 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 05:16:05,127 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 05:16:05,127 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 05:16:05,135 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 05:16:05,137 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 05:16:05,139 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 05:16:05,139 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 05:16:05,140 main DEBUG createAppenders(={Console}) 2024-12-05 05:16:05,140 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-05 05:16:05,141 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-05 05:16:05,141 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-05 05:16:05,141 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 05:16:05,142 main DEBUG OutputStream closed 2024-12-05 05:16:05,142 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 05:16:05,142 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 05:16:05,142 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-05 05:16:05,205 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 05:16:05,207 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 05:16:05,208 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 05:16:05,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 05:16:05,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 05:16:05,210 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 05:16:05,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 05:16:05,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 05:16:05,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 05:16:05,211 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 05:16:05,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 05:16:05,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 05:16:05,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 05:16:05,212 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 05:16:05,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 05:16:05,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 05:16:05,213 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 05:16:05,214 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 05:16:05,216 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 05:16:05,217 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-05 05:16:05,217 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 05:16:05,218 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-05T05:16:05,417 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0 2024-12-05 05:16:05,419 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 05:16:05,420 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T05:16:05,427 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-05T05:16:05,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T05:16:05,457 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c, deleteOnExit=true 2024-12-05T05:16:05,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-05T05:16:05,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/test.cache.data in system properties and HBase conf 2024-12-05T05:16:05,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T05:16:05,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/hadoop.log.dir in system properties and HBase conf 2024-12-05T05:16:05,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T05:16:05,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T05:16:05,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-05T05:16:05,539 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T05:16:05,623 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T05:16:05,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T05:16:05,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T05:16:05,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T05:16:05,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T05:16:05,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T05:16:05,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T05:16:05,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T05:16:05,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T05:16:05,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T05:16:05,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/nfs.dump.dir in system properties and HBase conf 2024-12-05T05:16:05,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/java.io.tmpdir in system properties and HBase conf 2024-12-05T05:16:05,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T05:16:05,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T05:16:05,631 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T05:16:06,794 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T05:16:06,889 INFO [Time-limited test {}] log.Log(170): Logging initialized @2418ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T05:16:06,977 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T05:16:07,048 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T05:16:07,079 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T05:16:07,080 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T05:16:07,082 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T05:16:07,099 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T05:16:07,102 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/hadoop.log.dir/,AVAILABLE} 2024-12-05T05:16:07,103 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T05:16:07,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3f7463d{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/java.io.tmpdir/jetty-localhost-45529-hadoop-hdfs-3_4_1-tests_jar-_-any-1273309268134591713/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T05:16:07,345 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:45529} 2024-12-05T05:16:07,346 INFO [Time-limited test {}] server.Server(415): Started @2876ms 2024-12-05T05:16:08,056 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T05:16:08,065 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T05:16:08,066 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T05:16:08,067 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T05:16:08,067 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-05T05:16:08,068 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@15b00a0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/hadoop.log.dir/,AVAILABLE} 2024-12-05T05:16:08,068 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3ad6a89f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T05:16:08,209 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@70061d7e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/java.io.tmpdir/jetty-localhost-37417-hadoop-hdfs-3_4_1-tests_jar-_-any-15852027833965956785/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T05:16:08,210 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@70195653{HTTP/1.1, (http/1.1)}{localhost:37417} 2024-12-05T05:16:08,210 INFO [Time-limited test {}] server.Server(415): Started @3741ms 2024-12-05T05:16:08,270 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T05:16:09,360 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/data/data1/current/BP-896134436-172.17.0.3-1733375766197/current, will proceed with Du for space computation calculation, 2024-12-05T05:16:09,360 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/data/data2/current/BP-896134436-172.17.0.3-1733375766197/current, will proceed with Du for space computation calculation, 2024-12-05T05:16:09,399 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T05:16:09,449 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x28d6685d5eec25b9 with lease ID 0x5e101e1305abf4e0: Processing first storage report for DS-a42e5228-c336-4525-82c5-b0a1aeb8f3c4 from datanode DatanodeRegistration(127.0.0.1:35739, datanodeUuid=00dd1f27-ceaa-4272-b9af-de2e158f6264, infoPort=41059, infoSecurePort=0, ipcPort=39113, storageInfo=lv=-57;cid=testClusterID;nsid=518435689;c=1733375766197) 2024-12-05T05:16:09,451 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x28d6685d5eec25b9 with lease ID 0x5e101e1305abf4e0: from storage DS-a42e5228-c336-4525-82c5-b0a1aeb8f3c4 node DatanodeRegistration(127.0.0.1:35739, datanodeUuid=00dd1f27-ceaa-4272-b9af-de2e158f6264, infoPort=41059, infoSecurePort=0, ipcPort=39113, storageInfo=lv=-57;cid=testClusterID;nsid=518435689;c=1733375766197), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-05T05:16:09,452 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x28d6685d5eec25b9 with lease ID 0x5e101e1305abf4e0: Processing first storage report for DS-a963c76d-c41b-417f-833e-ea96eaa597ca from datanode DatanodeRegistration(127.0.0.1:35739, datanodeUuid=00dd1f27-ceaa-4272-b9af-de2e158f6264, infoPort=41059, infoSecurePort=0, ipcPort=39113, storageInfo=lv=-57;cid=testClusterID;nsid=518435689;c=1733375766197) 2024-12-05T05:16:09,452 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x28d6685d5eec25b9 with lease ID 0x5e101e1305abf4e0: from storage DS-a963c76d-c41b-417f-833e-ea96eaa597ca node DatanodeRegistration(127.0.0.1:35739, datanodeUuid=00dd1f27-ceaa-4272-b9af-de2e158f6264, infoPort=41059, infoSecurePort=0, ipcPort=39113, storageInfo=lv=-57;cid=testClusterID;nsid=518435689;c=1733375766197), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T05:16:09,520 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0 2024-12-05T05:16:09,596 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/zookeeper_0, clientPort=57337, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T05:16:09,608 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57337 2024-12-05T05:16:09,623 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T05:16:09,627 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T05:16:09,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741825_1001 (size=7) 2024-12-05T05:16:10,288 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 with version=8 2024-12-05T05:16:10,288 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/hbase-staging 2024-12-05T05:16:10,392 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T05:16:10,693 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8edcaf2ecc84:0 server-side Connection retries=45 2024-12-05T05:16:10,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T05:16:10,704 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T05:16:10,709 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T05:16:10,709 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T05:16:10,709 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T05:16:10,867 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-05T05:16:10,938 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T05:16:10,950 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T05:16:10,955 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T05:16:10,985 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 5710 (auto-detected) 2024-12-05T05:16:10,987 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-05T05:16:11,010 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46695 2024-12-05T05:16:11,037 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46695 connecting to ZooKeeper ensemble=127.0.0.1:57337 2024-12-05T05:16:11,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:466950x0, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T05:16:11,169 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46695-0x101a3eb37930000 connected 2024-12-05T05:16:11,262 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T05:16:11,265 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T05:16:11,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T05:16:11,289 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426, hbase.cluster.distributed=false 2024-12-05T05:16:11,320 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T05:16:11,328 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46695 2024-12-05T05:16:11,331 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46695 2024-12-05T05:16:11,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46695 2024-12-05T05:16:11,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46695 2024-12-05T05:16:11,332 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46695 2024-12-05T05:16:11,437 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8edcaf2ecc84:0 server-side Connection retries=45 2024-12-05T05:16:11,439 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T05:16:11,439 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T05:16:11,439 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T05:16:11,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T05:16:11,440 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T05:16:11,443 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T05:16:11,453 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T05:16:11,454 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36233 2024-12-05T05:16:11,456 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36233 connecting to ZooKeeper ensemble=127.0.0.1:57337 2024-12-05T05:16:11,458 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T05:16:11,463 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T05:16:11,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:362330x0, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T05:16:11,488 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:362330x0, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T05:16:11,489 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36233-0x101a3eb37930001 connected 2024-12-05T05:16:11,493 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T05:16:11,509 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T05:16:11,512 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T05:16:11,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T05:16:11,540 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36233 2024-12-05T05:16:11,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36233 2024-12-05T05:16:11,545 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36233 2024-12-05T05:16:11,548 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36233 2024-12-05T05:16:11,549 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36233 2024-12-05T05:16:11,569 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8edcaf2ecc84:46695 2024-12-05T05:16:11,570 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:11,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T05:16:11,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T05:16:11,589 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:11,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:11,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T05:16:11,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:11,636 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T05:16:11,647 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8edcaf2ecc84,46695,1733375770505 from backup master directory 2024-12-05T05:16:11,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:11,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T05:16:11,661 WARN [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T05:16:11,662 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:11,665 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T05:16:11,667 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T05:16:11,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T05:16:11,753 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/hbase.id] with ID: 749228af-e9f8-4a64-bfdc-0b8e6f58fa86 2024-12-05T05:16:11,754 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/.tmp/hbase.id 2024-12-05T05:16:11,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741826_1002 (size=42) 2024-12-05T05:16:12,175 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/.tmp/hbase.id]:[hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/hbase.id] 2024-12-05T05:16:12,222 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T05:16:12,227 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-05T05:16:12,245 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-05T05:16:12,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:12,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:12,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741827_1003 (size=196) 2024-12-05T05:16:12,335 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:16:12,337 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T05:16:12,353 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:12,358 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T05:16:12,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741828_1004 (size=1189) 2024-12-05T05:16:12,438 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store 2024-12-05T05:16:12,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741829_1005 (size=34) 2024-12-05T05:16:12,867 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T05:16:12,870 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:12,872 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T05:16:12,872 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T05:16:12,872 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T05:16:12,875 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-12-05T05:16:12,875 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T05:16:12,875 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T05:16:12,877 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733375772872Disabling compacts and flushes for region at 1733375772872Disabling writes for close at 1733375772875 (+3 ms)Writing region close event to WAL at 1733375772875Closed at 1733375772875 2024-12-05T05:16:12,881 WARN [master/8edcaf2ecc84:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/.initializing 2024-12-05T05:16:12,882 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/WALs/8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:12,897 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T05:16:12,918 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8edcaf2ecc84%2C46695%2C1733375770505, suffix=, logDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/WALs/8edcaf2ecc84,46695,1733375770505, archiveDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/oldWALs, maxLogs=10 2024-12-05T05:16:12,949 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/WALs/8edcaf2ecc84,46695,1733375770505/8edcaf2ecc84%2C46695%2C1733375770505.1733375772925, exclude list is [], retry=0 2024-12-05T05:16:12,966 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35739,DS-a42e5228-c336-4525-82c5-b0a1aeb8f3c4,DISK] 2024-12-05T05:16:12,969 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T05:16:13,010 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/WALs/8edcaf2ecc84,46695,1733375770505/8edcaf2ecc84%2C46695%2C1733375770505.1733375772925 2024-12-05T05:16:13,011 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059)] 2024-12-05T05:16:13,012 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:16:13,012 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:13,017 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,018 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,060 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T05:16:13,092 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:13,095 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:13,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T05:16:13,101 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:13,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:13,102 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T05:16:13,107 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:13,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:13,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T05:16:13,112 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:13,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:13,114 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,118 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,120 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,126 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,127 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,130 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T05:16:13,137 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T05:16:13,145 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T05:16:13,147 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67746842, jitterRate=0.00950661301612854}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T05:16:13,153 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733375773032Initializing all the Stores at 1733375773038 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733375773039 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375773039Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375773040 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375773040Cleaning up temporary data from old regions at 1733375773127 (+87 ms)Region opened successfully at 1733375773153 (+26 ms) 2024-12-05T05:16:13,155 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T05:16:13,198 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecf0699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8edcaf2ecc84/172.17.0.3:0 2024-12-05T05:16:13,240 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-05T05:16:13,255 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T05:16:13,255 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T05:16:13,259 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T05:16:13,261 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T05:16:13,268 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-05T05:16:13,268 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T05:16:13,304 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T05:16:13,315 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T05:16:13,364 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-05T05:16:13,367 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T05:16:13,369 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T05:16:13,376 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-05T05:16:13,380 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T05:16:13,384 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T05:16:13,395 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-05T05:16:13,397 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T05:16:13,406 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T05:16:13,424 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T05:16:13,438 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T05:16:13,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T05:16:13,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T05:16:13,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:13,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:13,451 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8edcaf2ecc84,46695,1733375770505, sessionid=0x101a3eb37930000, setting cluster-up flag (Was=false) 2024-12-05T05:16:13,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:13,480 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:13,511 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T05:16:13,514 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:13,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:13,533 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:13,564 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T05:16:13,567 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:13,581 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-05T05:16:13,688 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(746): ClusterId : 749228af-e9f8-4a64-bfdc-0b8e6f58fa86 2024-12-05T05:16:13,696 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T05:16:13,730 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T05:16:13,730 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T05:16:13,741 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-05T05:16:13,749 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T05:16:13,750 DEBUG [RS:0;8edcaf2ecc84:36233 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c3211ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8edcaf2ecc84/172.17.0.3:0 2024-12-05T05:16:13,755 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-05T05:16:13,766 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T05:16:13,775 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8edcaf2ecc84:36233 2024-12-05T05:16:13,779 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-05T05:16:13,780 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-05T05:16:13,775 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8edcaf2ecc84,46695,1733375770505 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T05:16:13,780 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-05T05:16:13,791 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(2659): reportForDuty to master=8edcaf2ecc84,46695,1733375770505 with port=36233, startcode=1733375771401 2024-12-05T05:16:13,796 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8edcaf2ecc84:0, corePoolSize=5, maxPoolSize=5 2024-12-05T05:16:13,796 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8edcaf2ecc84:0, corePoolSize=5, maxPoolSize=5 2024-12-05T05:16:13,799 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8edcaf2ecc84:0, corePoolSize=5, maxPoolSize=5 2024-12-05T05:16:13,800 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8edcaf2ecc84:0, corePoolSize=5, maxPoolSize=5 2024-12-05T05:16:13,800 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8edcaf2ecc84:0, corePoolSize=10, maxPoolSize=10 2024-12-05T05:16:13,801 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:13,801 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8edcaf2ecc84:0, corePoolSize=2, maxPoolSize=2 2024-12-05T05:16:13,801 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:13,809 DEBUG [RS:0;8edcaf2ecc84:36233 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T05:16:13,830 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T05:16:13,831 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-05T05:16:13,837 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733375803836 2024-12-05T05:16:13,839 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:13,840 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T05:16:13,846 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T05:16:13,848 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T05:16:13,864 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T05:16:13,865 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T05:16:13,866 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T05:16:13,866 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T05:16:13,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741831_1007 (size=1321) 2024-12-05T05:16:13,880 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:13,889 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T05:16:13,896 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35105, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T05:16:13,896 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T05:16:13,897 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T05:16:13,900 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T05:16:13,901 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T05:16:13,906 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8edcaf2ecc84:0:becomeActiveMaster-HFileCleaner.large.0-1733375773904,5,FailOnTimeoutGroup] 2024-12-05T05:16:13,914 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46695 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:13,917 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8edcaf2ecc84:0:becomeActiveMaster-HFileCleaner.small.0-1733375773907,5,FailOnTimeoutGroup] 2024-12-05T05:16:13,917 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:13,918 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T05:16:13,919 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:13,920 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:13,963 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-05T05:16:13,963 WARN [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-05T05:16:14,064 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(2659): reportForDuty to master=8edcaf2ecc84,46695,1733375770505 with port=36233, startcode=1733375771401 2024-12-05T05:16:14,066 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46695 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:14,069 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46695 {}] master.ServerManager(517): Registering regionserver=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:14,076 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 2024-12-05T05:16:14,077 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:38835 2024-12-05T05:16:14,077 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-05T05:16:14,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T05:16:14,091 DEBUG [RS:0;8edcaf2ecc84:36233 {}] zookeeper.ZKUtil(111): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:14,091 WARN [RS:0;8edcaf2ecc84:36233 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T05:16:14,092 INFO [RS:0;8edcaf2ecc84:36233 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T05:16:14,092 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/WALs/8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:14,094 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8edcaf2ecc84,36233,1733375771401] 2024-12-05T05:16:14,124 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T05:16:14,142 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T05:16:14,149 INFO [RS:0;8edcaf2ecc84:36233 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T05:16:14,149 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,151 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-05T05:16:14,156 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-05T05:16:14,158 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,159 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,159 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,159 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,159 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,160 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,160 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8edcaf2ecc84:0, corePoolSize=2, maxPoolSize=2 2024-12-05T05:16:14,160 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,160 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,160 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,161 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,161 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,161 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8edcaf2ecc84:0, corePoolSize=1, maxPoolSize=1 2024-12-05T05:16:14,161 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8edcaf2ecc84:0, corePoolSize=3, maxPoolSize=3 2024-12-05T05:16:14,162 DEBUG [RS:0;8edcaf2ecc84:36233 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0, corePoolSize=3, maxPoolSize=3 2024-12-05T05:16:14,163 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,163 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,163 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,163 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,163 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,163 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=8edcaf2ecc84,36233,1733375771401-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T05:16:14,192 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T05:16:14,194 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=8edcaf2ecc84,36233,1733375771401-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,195 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,195 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.Replication(171): 8edcaf2ecc84,36233,1733375771401 started 2024-12-05T05:16:14,221 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:14,221 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1482): Serving as 8edcaf2ecc84,36233,1733375771401, RpcServer on 8edcaf2ecc84/172.17.0.3:36233, sessionid=0x101a3eb37930001 2024-12-05T05:16:14,222 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T05:16:14,222 DEBUG [RS:0;8edcaf2ecc84:36233 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:14,222 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8edcaf2ecc84,36233,1733375771401' 2024-12-05T05:16:14,223 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T05:16:14,223 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T05:16:14,224 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T05:16:14,225 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T05:16:14,225 DEBUG [RS:0;8edcaf2ecc84:36233 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:14,225 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8edcaf2ecc84,36233,1733375771401' 2024-12-05T05:16:14,225 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T05:16:14,226 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T05:16:14,227 DEBUG [RS:0;8edcaf2ecc84:36233 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T05:16:14,227 INFO [RS:0;8edcaf2ecc84:36233 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T05:16:14,227 INFO [RS:0;8edcaf2ecc84:36233 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T05:16:14,284 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-05T05:16:14,285 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 2024-12-05T05:16:14,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741832_1008 (size=32) 2024-12-05T05:16:14,333 INFO [RS:0;8edcaf2ecc84:36233 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T05:16:14,336 INFO [RS:0;8edcaf2ecc84:36233 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8edcaf2ecc84%2C36233%2C1733375771401, suffix=, logDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/WALs/8edcaf2ecc84,36233,1733375771401, archiveDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/oldWALs, maxLogs=32 2024-12-05T05:16:14,352 DEBUG [RS:0;8edcaf2ecc84:36233 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/WALs/8edcaf2ecc84,36233,1733375771401/8edcaf2ecc84%2C36233%2C1733375771401.1733375774339, exclude list is [], retry=0 2024-12-05T05:16:14,358 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35739,DS-a42e5228-c336-4525-82c5-b0a1aeb8f3c4,DISK] 2024-12-05T05:16:14,363 INFO [RS:0;8edcaf2ecc84:36233 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/WALs/8edcaf2ecc84,36233,1733375771401/8edcaf2ecc84%2C36233%2C1733375771401.1733375774339 2024-12-05T05:16:14,364 DEBUG [RS:0;8edcaf2ecc84:36233 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059)] 2024-12-05T05:16:14,698 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:14,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T05:16:14,704 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T05:16:14,705 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:14,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:14,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T05:16:14,709 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T05:16:14,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:14,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:14,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T05:16:14,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T05:16:14,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:14,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:14,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T05:16:14,720 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T05:16:14,720 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:14,721 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:14,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T05:16:14,724 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740 2024-12-05T05:16:14,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740 2024-12-05T05:16:14,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T05:16:14,729 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T05:16:14,730 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:16:14,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T05:16:14,739 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T05:16:14,742 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60841311, jitterRate=-0.09339381754398346}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:16:14,744 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733375774698Initializing all the Stores at 1733375774701 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733375774701Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733375774701Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375774701Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733375774701Cleaning up temporary data from old regions at 1733375774729 (+28 ms)Region opened successfully at 1733375774744 (+15 ms) 2024-12-05T05:16:14,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T05:16:14,745 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T05:16:14,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T05:16:14,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T05:16:14,745 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T05:16:14,747 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T05:16:14,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733375774745Disabling compacts and flushes for region at 1733375774745Disabling writes for close at 1733375774745Writing region close event to WAL at 1733375774747 (+2 ms)Closed at 1733375774747 2024-12-05T05:16:14,752 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T05:16:14,752 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-05T05:16:14,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T05:16:14,772 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T05:16:14,775 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T05:16:14,929 DEBUG [8edcaf2ecc84:46695 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T05:16:14,941 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:14,947 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8edcaf2ecc84,36233,1733375771401, state=OPENING 2024-12-05T05:16:15,058 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T05:16:15,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:15,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:16:15,070 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T05:16:15,071 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T05:16:15,072 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T05:16:15,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:16:15,254 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T05:16:15,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40795, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T05:16:15,286 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-05T05:16:15,287 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T05:16:15,287 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T05:16:15,291 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8edcaf2ecc84%2C36233%2C1733375771401.meta, suffix=.meta, logDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/WALs/8edcaf2ecc84,36233,1733375771401, archiveDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/oldWALs, maxLogs=32 2024-12-05T05:16:15,314 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/WALs/8edcaf2ecc84,36233,1733375771401/8edcaf2ecc84%2C36233%2C1733375771401.meta.1733375775293.meta, exclude list is [], retry=0 2024-12-05T05:16:15,318 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35739,DS-a42e5228-c336-4525-82c5-b0a1aeb8f3c4,DISK] 2024-12-05T05:16:15,322 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/WALs/8edcaf2ecc84,36233,1733375771401/8edcaf2ecc84%2C36233%2C1733375771401.meta.1733375775293.meta 2024-12-05T05:16:15,322 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41059:41059)] 2024-12-05T05:16:15,323 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:16:15,325 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T05:16:15,328 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T05:16:15,333 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T05:16:15,338 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T05:16:15,339 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:15,339 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-05T05:16:15,339 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-05T05:16:15,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T05:16:15,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T05:16:15,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:15,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:15,348 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-05T05:16:15,350 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-05T05:16:15,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:15,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:15,352 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T05:16:15,355 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T05:16:15,355 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:15,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:15,356 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T05:16:15,358 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T05:16:15,358 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:15,359 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T05:16:15,359 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-05T05:16:15,361 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740 2024-12-05T05:16:15,364 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740 2024-12-05T05:16:15,368 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-05T05:16:15,368 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-05T05:16:15,369 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:16:15,372 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-05T05:16:15,375 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72621704, jitterRate=0.08214771747589111}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:16:15,375 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-05T05:16:15,377 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733375775340Writing region info on filesystem at 1733375775340Initializing all the Stores at 1733375775342 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733375775342Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733375775344 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375775344Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733375775344Cleaning up temporary data from old regions at 1733375775368 (+24 ms)Running coprocessor post-open hooks at 1733375775375 (+7 ms)Region opened successfully at 1733375775377 (+2 ms) 2024-12-05T05:16:15,386 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733375775243 2024-12-05T05:16:15,399 DEBUG [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T05:16:15,399 INFO [RS_OPEN_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-05T05:16:15,401 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:15,404 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8edcaf2ecc84,36233,1733375771401, state=OPEN 2024-12-05T05:16:15,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T05:16:15,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T05:16:15,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T05:16:15,500 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T05:16:15,500 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:15,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T05:16:15,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8edcaf2ecc84,36233,1733375771401 in 426 msec 2024-12-05T05:16:15,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T05:16:15,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 750 msec 2024-12-05T05:16:15,517 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-05T05:16:15,517 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-05T05:16:15,537 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:15,539 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:15,563 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:15,565 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39859, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:15,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.9400 sec 2024-12-05T05:16:15,586 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733375775586, completionTime=-1 2024-12-05T05:16:15,589 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T05:16:15,589 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-05T05:16:15,616 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-05T05:16:15,616 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733375835616 2024-12-05T05:16:15,616 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733375895616 2024-12-05T05:16:15,616 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-12-05T05:16:15,619 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8edcaf2ecc84,46695,1733375770505-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:15,619 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8edcaf2ecc84,46695,1733375770505-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:15,619 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8edcaf2ecc84,46695,1733375770505-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:15,621 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8edcaf2ecc84:46695, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:15,621 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:15,622 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:15,628 DEBUG [master/8edcaf2ecc84:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-05T05:16:15,657 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.995sec 2024-12-05T05:16:15,659 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T05:16:15,661 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T05:16:15,662 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T05:16:15,663 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T05:16:15,663 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T05:16:15,664 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8edcaf2ecc84,46695,1733375770505-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T05:16:15,665 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8edcaf2ecc84,46695,1733375770505-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T05:16:15,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9654c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:15,675 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T05:16:15,675 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T05:16:15,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:15,684 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:15,698 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-05T05:16:15,699 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T05:16:15,699 INFO [master/8edcaf2ecc84:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8edcaf2ecc84,46695,1733375770505-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T05:16:15,701 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:15,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:15,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56004324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:15,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:15,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:15,722 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:15,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7449b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:15,727 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:15,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:15,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38628, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:15,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:15,792 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=683, ProcessCount=11, AvailableMemoryMB=3121 2024-12-05T05:16:15,805 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T05:16:15,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:15,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3108930d 2024-12-05T05:16:15,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T05:16:15,825 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T05:16:15,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:16:15,831 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:16:15,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:15,844 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T05:16:15,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-05T05:16:15,847 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:15,850 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T05:16:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T05:16:15,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741835_1011 (size=1061) 2024-12-05T05:16:15,946 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 2024-12-05T05:16:15,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741836_1012 (size=53) 2024-12-05T05:16:15,966 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:15,966 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6ed0ce7f5160651ec0c7956ddfe8e534, disabling compactions & flushes 2024-12-05T05:16:15,966 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:15,966 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:15,967 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. after waiting 0 ms 2024-12-05T05:16:15,967 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:15,967 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:15,967 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ed0ce7f5160651ec0c7956ddfe8e534: Waiting for close lock at 1733375775966Disabling compacts and flushes for region at 1733375775966Disabling writes for close at 1733375775967 (+1 ms)Writing region close event to WAL at 1733375775967Closed at 1733375775967 2024-12-05T05:16:15,970 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T05:16:15,976 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733375775970"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733375775970"}]},"ts":"1733375775970"} 2024-12-05T05:16:15,981 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T05:16:15,983 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T05:16:15,987 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375775984"}]},"ts":"1733375775984"} 2024-12-05T05:16:15,992 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T05:16:15,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T05:16:15,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, ASSIGN}] 2024-12-05T05:16:15,997 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, ASSIGN 2024-12-05T05:16:16,000 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, ASSIGN; state=OFFLINE, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=false 2024-12-05T05:16:16,152 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ed0ce7f5160651ec0c7956ddfe8e534, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:16,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, ASSIGN because future has completed 2024-12-05T05:16:16,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:16:16,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T05:16:16,320 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:16,321 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:16:16,321 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,321 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:16,321 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,322 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,328 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,333 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:16,334 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed0ce7f5160651ec0c7956ddfe8e534 columnFamilyName A 2024-12-05T05:16:16,334 DEBUG [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:16,336 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.HStore(327): Store=6ed0ce7f5160651ec0c7956ddfe8e534/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:16,336 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,342 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:16,343 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed0ce7f5160651ec0c7956ddfe8e534 columnFamilyName B 2024-12-05T05:16:16,343 DEBUG [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:16,345 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.HStore(327): Store=6ed0ce7f5160651ec0c7956ddfe8e534/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:16,345 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,348 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:16,349 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ed0ce7f5160651ec0c7956ddfe8e534 columnFamilyName C 2024-12-05T05:16:16,349 DEBUG [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:16,350 INFO [StoreOpener-6ed0ce7f5160651ec0c7956ddfe8e534-1 {}] regionserver.HStore(327): Store=6ed0ce7f5160651ec0c7956ddfe8e534/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:16,351 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:16,351 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,356 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,360 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,364 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,364 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,369 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:16:16,373 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,384 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T05:16:16,386 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6ed0ce7f5160651ec0c7956ddfe8e534; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67294660, jitterRate=0.002768576145172119}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:16:16,386 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,387 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6ed0ce7f5160651ec0c7956ddfe8e534: Running coprocessor pre-open hook at 1733375776322Writing region info on filesystem at 1733375776322Initializing all the Stores at 1733375776324 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375776324Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375776328 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375776328Cleaning up temporary data from old regions at 1733375776364 (+36 ms)Running coprocessor post-open hooks at 1733375776386 (+22 ms)Region opened successfully at 1733375776387 (+1 ms) 2024-12-05T05:16:16,390 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., pid=6, masterSystemTime=1733375776313 2024-12-05T05:16:16,395 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:16,395 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:16,396 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ed0ce7f5160651ec0c7956ddfe8e534, regionState=OPEN, openSeqNum=2, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:16,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:16:16,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T05:16:16,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 in 248 msec 2024-12-05T05:16:16,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T05:16:16,418 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, ASSIGN in 417 msec 2024-12-05T05:16:16,421 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T05:16:16,421 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375776421"}]},"ts":"1733375776421"} 2024-12-05T05:16:16,425 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T05:16:16,428 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T05:16:16,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 593 msec 2024-12-05T05:16:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-05T05:16:16,515 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:16,522 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13af8352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,523 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,527 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ddcdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,534 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6924ee0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,538 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,544 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,547 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T05:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6d6e054b 2024-12-05T05:16:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T05:16:16,569 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T05:16:16,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd75e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,587 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2555cfdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,588 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,590 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,592 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@90ab23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,595 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d2118b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,595 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,597 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5c6971, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,602 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73a084b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b60dcf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,608 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a5a580f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,615 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46fadb48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@323acfb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,626 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ac9955, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72f860d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3f6ca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,642 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ae98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,646 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54370, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e8180ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a68df1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,650 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,650 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,652 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@201fc254, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a8a735f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@667ba1d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,665 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,665 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,667 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0a534c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,669 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,672 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,677 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58dec3db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20e35624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,681 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,682 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@388f703a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,686 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,687 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7299e0d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c99741, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:16,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:16,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:16,695 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bb69f80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:16,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:16,698 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:16,698 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54458, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:16,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c19ce67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:16,709 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,710 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,724 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,724 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,734 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,751 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,756 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,759 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,761 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,765 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,779 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,795 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,797 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,798 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,800 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:16,814 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-05T05:16:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:16,824 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:16,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T05:16:16,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:16:16,836 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:16,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:16,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:16:16,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:16,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:16,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:16,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:16,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:16,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T05:16:16,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/8fa89a669e48410298b67065581e8463 is 50, key is test_row_0/A:col10/1733375776820/Put/seqid=0 2024-12-05T05:16:17,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741837_1013 (size=12001) 2024-12-05T05:16:17,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T05:16:17,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:17,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375837017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375837022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375837028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375837054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375837055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T05:16:17,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375837186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375837189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375837184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375837194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375837195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,215 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T05:16:17,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:17,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T05:16:17,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375837405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375837406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375837406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,412 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/8fa89a669e48410298b67065581e8463 2024-12-05T05:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:17,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375837423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375837424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,440 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T05:16:17,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T05:16:17,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:17,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/9ec8ee513f2a41babcef1b4b8da4f3b2 is 50, key is test_row_0/B:col10/1733375776820/Put/seqid=0 2024-12-05T05:16:17,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375837714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375837714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,724 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375837723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741838_1014 (size=12001) 2024-12-05T05:16:17,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/9ec8ee513f2a41babcef1b4b8da4f3b2 2024-12-05T05:16:17,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T05:16:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:17,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375837743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,760 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,761 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375837754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:17,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:17,829 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/5141392d483c416bb5f25bdf5ecd6be1 is 50, key is test_row_0/C:col10/1733375776820/Put/seqid=0 2024-12-05T05:16:17,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741839_1015 (size=12001) 2024-12-05T05:16:17,888 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/5141392d483c416bb5f25bdf5ecd6be1 2024-12-05T05:16:17,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T05:16:17,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:17,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:17,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/8fa89a669e48410298b67065581e8463 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8fa89a669e48410298b67065581e8463 2024-12-05T05:16:17,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T05:16:17,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8fa89a669e48410298b67065581e8463, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T05:16:18,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/9ec8ee513f2a41babcef1b4b8da4f3b2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9ec8ee513f2a41babcef1b4b8da4f3b2 2024-12-05T05:16:18,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9ec8ee513f2a41babcef1b4b8da4f3b2, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T05:16:18,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/5141392d483c416bb5f25bdf5ecd6be1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5141392d483c416bb5f25bdf5ecd6be1 2024-12-05T05:16:18,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5141392d483c416bb5f25bdf5ecd6be1, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T05:16:18,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1233ms, sequenceid=14, compaction requested=false 2024-12-05T05:16:18,075 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T05:16:18,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:18,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-05T05:16:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:18,096 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:16:18,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:18,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:18,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:18,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:18,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:18,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:18,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/2f97f673babb4df49d53b49eeba6410d is 50, key is test_row_0/A:col10/1733375777020/Put/seqid=0 2024-12-05T05:16:18,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741840_1016 (size=12001) 2024-12-05T05:16:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:18,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:18,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375838308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375838315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375838321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375838334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375838351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375838460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375838465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375838468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375838474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375838484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,590 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/2f97f673babb4df49d53b49eeba6410d 2024-12-05T05:16:18,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7ae9eb650ff847bbb4d096d06e0dd465 is 50, key is test_row_0/B:col10/1733375777020/Put/seqid=0 2024-12-05T05:16:18,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741841_1017 (size=12001) 2024-12-05T05:16:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375838677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,685 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7ae9eb650ff847bbb4d096d06e0dd465 2024-12-05T05:16:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375838692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375838698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375838701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375838704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:18,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/776525c7039941b1aac36c40f6db432e is 50, key is test_row_0/C:col10/1733375777020/Put/seqid=0 2024-12-05T05:16:18,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741842_1018 (size=12001) 2024-12-05T05:16:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T05:16:19,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375839001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375839014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375839030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375839037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375839041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,199 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/776525c7039941b1aac36c40f6db432e 2024-12-05T05:16:19,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/2f97f673babb4df49d53b49eeba6410d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2f97f673babb4df49d53b49eeba6410d 2024-12-05T05:16:19,252 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2f97f673babb4df49d53b49eeba6410d, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T05:16:19,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7ae9eb650ff847bbb4d096d06e0dd465 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7ae9eb650ff847bbb4d096d06e0dd465 2024-12-05T05:16:19,274 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7ae9eb650ff847bbb4d096d06e0dd465, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T05:16:19,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/776525c7039941b1aac36c40f6db432e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/776525c7039941b1aac36c40f6db432e 2024-12-05T05:16:19,301 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/776525c7039941b1aac36c40f6db432e, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T05:16:19,303 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1207ms, sequenceid=33, compaction requested=false 2024-12-05T05:16:19,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:19,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:19,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-05T05:16:19,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-05T05:16:19,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-05T05:16:19,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4770 sec 2024-12-05T05:16:19,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5120 sec 2024-12-05T05:16:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:19,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T05:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:19,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:19,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:19,574 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/e25d5714eacd46899d97de8489a784d2 is 50, key is test_row_0/A:col10/1733375778301/Put/seqid=0 2024-12-05T05:16:19,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375839629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,651 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375839632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375839633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375839642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375839648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741843_1019 (size=14341) 2024-12-05T05:16:19,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/e25d5714eacd46899d97de8489a784d2 2024-12-05T05:16:19,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/95dc830dd04c4e63b0caf0940256d6ab is 50, key is test_row_0/B:col10/1733375778301/Put/seqid=0 2024-12-05T05:16:19,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741844_1020 (size=12001) 2024-12-05T05:16:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375839782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375839783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375839776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375839783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375839785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:19,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:19,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375839994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:19,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375840009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375840010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375840003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,016 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375840011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,024 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,116 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T05:16:20,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/95dc830dd04c4e63b0caf0940256d6ab 2024-12-05T05:16:20,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0b40ba9c375f47e697419a2a286881f9 is 50, key is test_row_0/C:col10/1733375778301/Put/seqid=0 2024-12-05T05:16:20,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741845_1021 (size=12001) 2024-12-05T05:16:20,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375840303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375840326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375840332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375840335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375840336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0b40ba9c375f47e697419a2a286881f9 2024-12-05T05:16:20,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/e25d5714eacd46899d97de8489a784d2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e25d5714eacd46899d97de8489a784d2 2024-12-05T05:16:20,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e25d5714eacd46899d97de8489a784d2, entries=200, sequenceid=48, filesize=14.0 K 2024-12-05T05:16:20,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/95dc830dd04c4e63b0caf0940256d6ab as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/95dc830dd04c4e63b0caf0940256d6ab 2024-12-05T05:16:20,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/95dc830dd04c4e63b0caf0940256d6ab, entries=150, sequenceid=48, filesize=11.7 K 2024-12-05T05:16:20,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375840834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,844 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0b40ba9c375f47e697419a2a286881f9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0b40ba9c375f47e697419a2a286881f9 2024-12-05T05:16:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375840845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375840848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,861 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,864 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T05:16:20,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375840854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375840874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:20,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0b40ba9c375f47e697419a2a286881f9, entries=150, sequenceid=48, filesize=11.7 K 2024-12-05T05:16:20,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1371ms, sequenceid=48, compaction requested=true 2024-12-05T05:16:20,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:20,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:20,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:20,954 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T05:16:20,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:20,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:20,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:20,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T05:16:20,955 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T05:16:20,957 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:20,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T05:16:20,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T05:16:20,959 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T05:16:20,959 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T05:16:20,961 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:20,963 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:16:20,963 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T05:16:20,973 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:20,973 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:20,976 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:20,976 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:20,977 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:20,977 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:20,978 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9ec8ee513f2a41babcef1b4b8da4f3b2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7ae9eb650ff847bbb4d096d06e0dd465, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/95dc830dd04c4e63b0caf0940256d6ab] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=35.2 K 2024-12-05T05:16:20,978 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8fa89a669e48410298b67065581e8463, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2f97f673babb4df49d53b49eeba6410d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e25d5714eacd46899d97de8489a784d2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=37.4 K 2024-12-05T05:16:20,981 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ec8ee513f2a41babcef1b4b8da4f3b2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375776820 2024-12-05T05:16:20,982 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa89a669e48410298b67065581e8463, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375776820 2024-12-05T05:16:20,982 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae9eb650ff847bbb4d096d06e0dd465, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733375777009 2024-12-05T05:16:20,983 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 95dc830dd04c4e63b0caf0940256d6ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733375778301 2024-12-05T05:16:20,984 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f97f673babb4df49d53b49eeba6410d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733375777009 2024-12-05T05:16:20,988 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25d5714eacd46899d97de8489a784d2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733375778301 2024-12-05T05:16:21,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-05T05:16:21,004 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:21,005 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-05T05:16:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T05:16:21,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:21,011 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:21,011 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:21,053 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#9 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:21,056 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/5618428a273043f884493f65d4363ecf is 50, key is test_row_0/B:col10/1733375778301/Put/seqid=0 2024-12-05T05:16:21,066 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:21,067 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/6bd2f8c76ade4be2810f474674ea1310 is 50, key is test_row_0/A:col10/1733375778301/Put/seqid=0 2024-12-05T05:16:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T05:16:21,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741846_1022 (size=12104) 2024-12-05T05:16:21,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741847_1023 (size=12104) 2024-12-05T05:16:21,163 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/5618428a273043f884493f65d4363ecf as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/5618428a273043f884493f65d4363ecf 2024-12-05T05:16:21,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-05T05:16:21,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:21,170 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T05:16:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:21,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:21,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:21,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:21,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:21,181 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/6bd2f8c76ade4be2810f474674ea1310 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6bd2f8c76ade4be2810f474674ea1310 2024-12-05T05:16:21,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/7f2481826ea34bc59f184a136a7b0675 is 50, key is test_row_0/A:col10/1733375779610/Put/seqid=0 2024-12-05T05:16:21,203 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into 5618428a273043f884493f65d4363ecf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:21,203 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:21,203 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into 6bd2f8c76ade4be2810f474674ea1310(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:21,204 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:21,212 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375780954; duration=0sec 2024-12-05T05:16:21,219 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:21,219 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375780916; duration=0sec 2024-12-05T05:16:21,220 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:21,220 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:21,220 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:21,220 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:21,223 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:21,223 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:21,224 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:21,224 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5141392d483c416bb5f25bdf5ecd6be1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/776525c7039941b1aac36c40f6db432e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0b40ba9c375f47e697419a2a286881f9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=35.2 K 2024-12-05T05:16:21,240 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 5141392d483c416bb5f25bdf5ecd6be1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375776820 2024-12-05T05:16:21,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741848_1024 (size=12001) 2024-12-05T05:16:21,252 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/7f2481826ea34bc59f184a136a7b0675 2024-12-05T05:16:21,273 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 776525c7039941b1aac36c40f6db432e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733375777009 2024-12-05T05:16:21,276 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b40ba9c375f47e697419a2a286881f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733375778301 2024-12-05T05:16:21,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/f0485d2dc6d44c13a2422077dc1c0283 is 50, key is test_row_0/B:col10/1733375779610/Put/seqid=0 2024-12-05T05:16:21,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T05:16:21,334 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:21,335 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f3f8d8e303b84111958c898240f71642 is 50, key is test_row_0/C:col10/1733375778301/Put/seqid=0 2024-12-05T05:16:21,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741849_1025 (size=12001) 2024-12-05T05:16:21,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741850_1026 (size=12104) 2024-12-05T05:16:21,434 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f3f8d8e303b84111958c898240f71642 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f3f8d8e303b84111958c898240f71642 2024-12-05T05:16:21,459 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into f3f8d8e303b84111958c898240f71642(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:21,459 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:21,460 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375780955; duration=0sec 2024-12-05T05:16:21,460 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:21,460 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T05:16:21,793 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/f0485d2dc6d44c13a2422077dc1c0283 2024-12-05T05:16:21,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/5b03cedfe9cc4bee9d997d858d172c37 is 50, key is test_row_0/C:col10/1733375779610/Put/seqid=0 2024-12-05T05:16:21,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741851_1027 (size=12001) 2024-12-05T05:16:21,853 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/5b03cedfe9cc4bee9d997d858d172c37 2024-12-05T05:16:21,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:21,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/7f2481826ea34bc59f184a136a7b0675 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/7f2481826ea34bc59f184a136a7b0675 2024-12-05T05:16:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:21,895 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/7f2481826ea34bc59f184a136a7b0675, entries=150, sequenceid=65, filesize=11.7 K 2024-12-05T05:16:21,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/f0485d2dc6d44c13a2422077dc1c0283 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/f0485d2dc6d44c13a2422077dc1c0283 2024-12-05T05:16:21,915 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/f0485d2dc6d44c13a2422077dc1c0283, entries=150, sequenceid=65, filesize=11.7 K 2024-12-05T05:16:21,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/5b03cedfe9cc4bee9d997d858d172c37 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5b03cedfe9cc4bee9d997d858d172c37 2024-12-05T05:16:21,947 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5b03cedfe9cc4bee9d997d858d172c37, entries=150, sequenceid=65, filesize=11.7 K 2024-12-05T05:16:21,950 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=73.80 KB/75570 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 779ms, sequenceid=65, compaction requested=false 2024-12-05T05:16:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:21,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-05T05:16:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-05T05:16:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:21,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T05:16:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:21,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:21,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:21,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:21,962 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:21,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T05:16:21,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 949 msec 2024-12-05T05:16:21,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 968 msec 2024-12-05T05:16:21,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/30265cbd621649958a0b7efdd589c620 is 50, key is test_row_0/A:col10/1733375781955/Put/seqid=0 2024-12-05T05:16:22,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741852_1028 (size=16681) 2024-12-05T05:16:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375842035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375842040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375842046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375842051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375842052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-05T05:16:22,148 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:22,148 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-05T05:16:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T05:16:22,167 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:22,169 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:22,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:22,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375842169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375842171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375842184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375842184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375842193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T05:16:22,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:22,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375842384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,387 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375842389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,407 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/30265cbd621649958a0b7efdd589c620 2024-12-05T05:16:22,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375842405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375842405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375842405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/44ad257f09154aa49e60d245d86f3766 is 50, key is test_row_0/B:col10/1733375781955/Put/seqid=0 2024-12-05T05:16:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T05:16:22,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:22,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:22,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741853_1029 (size=12001) 2024-12-05T05:16:22,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:22,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:22,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375842696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375842713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375842723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375842724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375842724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:22,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T05:16:22,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:22,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:22,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/44ad257f09154aa49e60d245d86f3766 2024-12-05T05:16:22,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/c32e7ea3f28d4658af0dec9af2f8125f is 50, key is test_row_0/C:col10/1733375781955/Put/seqid=0 2024-12-05T05:16:22,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:22,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:22,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:22,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:22,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741854_1030 (size=12001) 2024-12-05T05:16:22,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/c32e7ea3f28d4658af0dec9af2f8125f 2024-12-05T05:16:23,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/30265cbd621649958a0b7efdd589c620 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/30265cbd621649958a0b7efdd589c620 2024-12-05T05:16:23,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/30265cbd621649958a0b7efdd589c620, entries=250, sequenceid=83, filesize=16.3 K 2024-12-05T05:16:23,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/44ad257f09154aa49e60d245d86f3766 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/44ad257f09154aa49e60d245d86f3766 2024-12-05T05:16:23,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/44ad257f09154aa49e60d245d86f3766, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T05:16:23,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/c32e7ea3f28d4658af0dec9af2f8125f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c32e7ea3f28d4658af0dec9af2f8125f 2024-12-05T05:16:23,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c32e7ea3f28d4658af0dec9af2f8125f, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T05:16:23,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:23,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:23,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1170ms, sequenceid=83, compaction requested=true 2024-12-05T05:16:23,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:23,131 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:23,133 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:23,133 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:23,133 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,134 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6bd2f8c76ade4be2810f474674ea1310, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/7f2481826ea34bc59f184a136a7b0675, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/30265cbd621649958a0b7efdd589c620] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=39.8 K 2024-12-05T05:16:23,142 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd2f8c76ade4be2810f474674ea1310, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733375778301 2024-12-05T05:16:23,144 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f2481826ea34bc59f184a136a7b0675, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733375779610 2024-12-05T05:16:23,145 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30265cbd621649958a0b7efdd589c620, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375781928 2024-12-05T05:16:23,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:23,160 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:23,162 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:23,162 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:23,162 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,162 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/5618428a273043f884493f65d4363ecf, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/f0485d2dc6d44c13a2422077dc1c0283, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/44ad257f09154aa49e60d245d86f3766] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=35.3 K 2024-12-05T05:16:23,163 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 5618428a273043f884493f65d4363ecf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733375778301 2024-12-05T05:16:23,164 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f0485d2dc6d44c13a2422077dc1c0283, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733375779610 2024-12-05T05:16:23,166 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ad257f09154aa49e60d245d86f3766, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375781934 2024-12-05T05:16:23,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:23,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:23,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:23,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:23,175 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#18 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:23,176 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/982b7d538d9244a383eab773087a09c3 is 50, key is test_row_0/A:col10/1733375781955/Put/seqid=0 2024-12-05T05:16:23,192 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:23,193 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/8838db0fc69d4c3b8ddf16e97b68763a is 50, key is test_row_0/B:col10/1733375781955/Put/seqid=0 2024-12-05T05:16:23,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741855_1031 (size=12207) 2024-12-05T05:16:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:23,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:23,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:23,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741856_1032 (size=12207) 2024-12-05T05:16:23,226 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/982b7d538d9244a383eab773087a09c3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/982b7d538d9244a383eab773087a09c3 2024-12-05T05:16:23,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/946352fb63484168bff636c2437ad7cd is 50, key is test_row_0/A:col10/1733375782009/Put/seqid=0 2024-12-05T05:16:23,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:23,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:23,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,297 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into 982b7d538d9244a383eab773087a09c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:23,298 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:23,298 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375783131; duration=0sec 2024-12-05T05:16:23,298 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:23,298 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:23,299 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:23,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T05:16:23,308 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/8838db0fc69d4c3b8ddf16e97b68763a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8838db0fc69d4c3b8ddf16e97b68763a 2024-12-05T05:16:23,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741857_1033 (size=14341) 2024-12-05T05:16:23,317 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:23,318 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:23,318 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,318 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f3f8d8e303b84111958c898240f71642, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5b03cedfe9cc4bee9d997d858d172c37, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c32e7ea3f28d4658af0dec9af2f8125f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=35.3 K 2024-12-05T05:16:23,321 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3f8d8e303b84111958c898240f71642, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733375778301 2024-12-05T05:16:23,329 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b03cedfe9cc4bee9d997d858d172c37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1733375779610 2024-12-05T05:16:23,331 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c32e7ea3f28d4658af0dec9af2f8125f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375781934 2024-12-05T05:16:23,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375843306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375843306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375843310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,341 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,342 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375843339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,346 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,361 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into 8838db0fc69d4c3b8ddf16e97b68763a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:23,361 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:23,361 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375783160; duration=0sec 2024-12-05T05:16:23,361 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:23,361 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:23,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375843359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,367 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,375 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:23,376 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/72c55c57017f46fc933e09a3897f4d4e is 50, key is test_row_0/C:col10/1733375781955/Put/seqid=0 2024-12-05T05:16:23,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741858_1034 (size=12207) 2024-12-05T05:16:23,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:23,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:23,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375843444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375843444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375843454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375843455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,463 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/72c55c57017f46fc933e09a3897f4d4e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/72c55c57017f46fc933e09a3897f4d4e 2024-12-05T05:16:23,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375843475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,509 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into 72c55c57017f46fc933e09a3897f4d4e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:23,509 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:23,509 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375783173; duration=0sec 2024-12-05T05:16:23,510 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:23,510 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:23,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:23,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375843663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375843663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375843663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,676 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375843664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375843683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/946352fb63484168bff636c2437ad7cd 2024-12-05T05:16:23,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:23,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/dcf82ae66fb24ccb9576f33e31e9a1df is 50, key is test_row_0/B:col10/1733375782009/Put/seqid=0 2024-12-05T05:16:23,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741859_1035 (size=12001) 2024-12-05T05:16:23,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:23,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:23,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:23,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:23,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375843985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375843985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:23,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:23,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375843985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:23,995 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:24,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375843995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:24,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:24,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375843995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:24,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/dcf82ae66fb24ccb9576f33e31e9a1df 2024-12-05T05:16:24,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a802a65425464bc78e3080ffbf6a958d is 50, key is test_row_0/C:col10/1733375782009/Put/seqid=0 2024-12-05T05:16:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T05:16:24,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741860_1036 (size=12001) 2024-12-05T05:16:24,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:24,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375844505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:24,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375844505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:24,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375844504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375844515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375844514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,516 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:24,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:24,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:24,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:24,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:24,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a802a65425464bc78e3080ffbf6a958d 2024-12-05T05:16:24,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/946352fb63484168bff636c2437ad7cd as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/946352fb63484168bff636c2437ad7cd 2024-12-05T05:16:24,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/946352fb63484168bff636c2437ad7cd, entries=200, sequenceid=102, filesize=14.0 K 2024-12-05T05:16:24,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/dcf82ae66fb24ccb9576f33e31e9a1df as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/dcf82ae66fb24ccb9576f33e31e9a1df 2024-12-05T05:16:24,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/dcf82ae66fb24ccb9576f33e31e9a1df, entries=150, sequenceid=102, filesize=11.7 K 2024-12-05T05:16:24,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a802a65425464bc78e3080ffbf6a958d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a802a65425464bc78e3080ffbf6a958d 2024-12-05T05:16:24,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a802a65425464bc78e3080ffbf6a958d, entries=150, sequenceid=102, filesize=11.7 K 2024-12-05T05:16:24,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1639ms, sequenceid=102, compaction requested=false 2024-12-05T05:16:24,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:24,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-05T05:16:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:24,891 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:16:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:24,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:24,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:24,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:24,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:24,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/b57f78a2e7ee4cb785c12bc3b23af83d is 50, key is test_row_0/A:col10/1733375783235/Put/seqid=0 2024-12-05T05:16:24,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741861_1037 (size=12001) 2024-12-05T05:16:24,947 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/b57f78a2e7ee4cb785c12bc3b23af83d 2024-12-05T05:16:24,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/9884f65f1eeb4e4b8dc1ad7f027a7697 is 50, key is test_row_0/B:col10/1733375783235/Put/seqid=0 2024-12-05T05:16:25,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741862_1038 (size=12001) 2024-12-05T05:16:25,021 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/9884f65f1eeb4e4b8dc1ad7f027a7697 2024-12-05T05:16:25,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0619bb7e48ad4622acfdfca45723ea3f is 50, key is test_row_0/C:col10/1733375783235/Put/seqid=0 2024-12-05T05:16:25,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741863_1039 (size=12001) 2024-12-05T05:16:25,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0619bb7e48ad4622acfdfca45723ea3f 2024-12-05T05:16:25,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/b57f78a2e7ee4cb785c12bc3b23af83d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b57f78a2e7ee4cb785c12bc3b23af83d 2024-12-05T05:16:25,137 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b57f78a2e7ee4cb785c12bc3b23af83d, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T05:16:25,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/9884f65f1eeb4e4b8dc1ad7f027a7697 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9884f65f1eeb4e4b8dc1ad7f027a7697 2024-12-05T05:16:25,182 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9884f65f1eeb4e4b8dc1ad7f027a7697, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T05:16:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0619bb7e48ad4622acfdfca45723ea3f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0619bb7e48ad4622acfdfca45723ea3f 2024-12-05T05:16:25,199 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0619bb7e48ad4622acfdfca45723ea3f, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T05:16:25,206 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 314ms, sequenceid=118, compaction requested=true 2024-12-05T05:16:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-05T05:16:25,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-05T05:16:25,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T05:16:25,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0400 sec 2024-12-05T05:16:25,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 3.0700 sec 2024-12-05T05:16:25,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:16:25,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:25,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:25,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:25,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:25,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:25,639 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:25,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/497b4682c46541409e59fb980dfb8465 is 50, key is test_row_0/A:col10/1733375785519/Put/seqid=0 2024-12-05T05:16:25,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741864_1040 (size=19071) 2024-12-05T05:16:25,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/497b4682c46541409e59fb980dfb8465 2024-12-05T05:16:25,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/899d832415594f3cabd4ff034825d625 is 50, key is test_row_0/B:col10/1733375785519/Put/seqid=0 2024-12-05T05:16:25,769 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733375785698, max timestamp=1733375785743Num uniques 859; 2024-12-05T05:16:25,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733375785698, max timestamp=1733375785743Num uniques 859; 2024-12-05T05:16:25,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T05:16:25,771 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T05:16:25,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733375785698, max timestamp=1733375785745Num uniques 859; 2024-12-05T05:16:25,780 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T05:16:25,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741865_1041 (size=12051) 2024-12-05T05:16:25,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/899d832415594f3cabd4ff034825d625 2024-12-05T05:16:25,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/7c828096b2154e219493c73389db2044 is 50, key is test_row_0/C:col10/1733375785519/Put/seqid=0 2024-12-05T05:16:25,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375845828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:25,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375845836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,856 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:25,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375845843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:25,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:25,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375845853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:25,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375845850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:25,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741866_1042 (size=12051) 2024-12-05T05:16:25,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/7c828096b2154e219493c73389db2044 2024-12-05T05:16:25,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/497b4682c46541409e59fb980dfb8465 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/497b4682c46541409e59fb980dfb8465 2024-12-05T05:16:25,940 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/497b4682c46541409e59fb980dfb8465, entries=300, sequenceid=129, filesize=18.6 K 2024-12-05T05:16:25,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/899d832415594f3cabd4ff034825d625 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/899d832415594f3cabd4ff034825d625 2024-12-05T05:16:25,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/899d832415594f3cabd4ff034825d625, entries=150, sequenceid=129, filesize=11.8 K 2024-12-05T05:16:25,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/7c828096b2154e219493c73389db2044 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7c828096b2154e219493c73389db2044 2024-12-05T05:16:25,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:25,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7c828096b2154e219493c73389db2044, entries=150, sequenceid=129, filesize=11.8 K 2024-12-05T05:16:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375845970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:25,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:25,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=126.28 KB/129315 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 359ms, sequenceid=129, compaction requested=true 2024-12-05T05:16:25,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:25,998 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:16:26,002 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57620 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:16:26,002 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:26,002 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,002 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/982b7d538d9244a383eab773087a09c3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/946352fb63484168bff636c2437ad7cd, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b57f78a2e7ee4cb785c12bc3b23af83d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/497b4682c46541409e59fb980dfb8465] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=56.3 K 2024-12-05T05:16:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:26,003 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 982b7d538d9244a383eab773087a09c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375781934 2024-12-05T05:16:26,004 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 946352fb63484168bff636c2437ad7cd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733375782009 2024-12-05T05:16:26,005 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57f78a2e7ee4cb785c12bc3b23af83d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733375783235 2024-12-05T05:16:26,006 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 497b4682c46541409e59fb980dfb8465, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375785519 2024-12-05T05:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:26,006 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:16:26,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:26,007 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:26,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=132.99 KB heapSize=382.48 KB 2024-12-05T05:16:26,011 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:16:26,012 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:26,012 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,012 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8838db0fc69d4c3b8ddf16e97b68763a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/dcf82ae66fb24ccb9576f33e31e9a1df, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9884f65f1eeb4e4b8dc1ad7f027a7697, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/899d832415594f3cabd4ff034825d625] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=47.1 K 2024-12-05T05:16:26,014 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8838db0fc69d4c3b8ddf16e97b68763a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375781934 2024-12-05T05:16:26,017 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf82ae66fb24ccb9576f33e31e9a1df, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733375782009 2024-12-05T05:16:26,019 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 9884f65f1eeb4e4b8dc1ad7f027a7697, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733375783235 2024-12-05T05:16:26,021 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 899d832415594f3cabd4ff034825d625, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375785519 2024-12-05T05:16:26,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375785824, max timestamp=1733375785997Num uniques 1009; 2024-12-05T05:16:26,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-05T05:16:26,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375785824, max timestamp=1733375785997Num uniques 1009; 2024-12-05T05:16:26,039 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-05T05:16:26,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375785824, max timestamp=1733375786005Num uniques 1009; 2024-12-05T05:16:26,048 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-05T05:16:26,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:26,065 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:26,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:26,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:26,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:26,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:26,071 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#30 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:26,072 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/2543ec8db930491ea275fe23a158bfda is 50, key is test_row_0/A:col10/1733375785519/Put/seqid=0 2024-12-05T05:16:26,080 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:26,081 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/8dc734efd8914e25b9a05cf71d6c163d is 50, key is test_row_0/B:col10/1733375785519/Put/seqid=0 2024-12-05T05:16:26,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/68c7e28a3f0d44c7b2eb6e99a347fe8c is 50, key is test_row_0/A:col10/1733375785997/Put/seqid=0 2024-12-05T05:16:26,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741867_1043 (size=12393) 2024-12-05T05:16:26,169 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/2543ec8db930491ea275fe23a158bfda as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2543ec8db930491ea275fe23a158bfda 2024-12-05T05:16:26,186 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into 2543ec8db930491ea275fe23a158bfda(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:26,186 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:26,186 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=12, startTime=1733375785998; duration=0sec 2024-12-05T05:16:26,187 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:26,187 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:26,187 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:16:26,195 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:16:26,195 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:26,195 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,195 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/72c55c57017f46fc933e09a3897f4d4e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a802a65425464bc78e3080ffbf6a958d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0619bb7e48ad4622acfdfca45723ea3f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7c828096b2154e219493c73389db2044] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=47.1 K 2024-12-05T05:16:26,196 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72c55c57017f46fc933e09a3897f4d4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375781934 2024-12-05T05:16:26,197 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a802a65425464bc78e3080ffbf6a958d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1733375782009 2024-12-05T05:16:26,198 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0619bb7e48ad4622acfdfca45723ea3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733375783235 2024-12-05T05:16:26,199 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c828096b2154e219493c73389db2044, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375785519 2024-12-05T05:16:26,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741868_1044 (size=12393) 2024-12-05T05:16:26,227 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/8dc734efd8914e25b9a05cf71d6c163d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8dc734efd8914e25b9a05cf71d6c163d 2024-12-05T05:16:26,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741869_1045 (size=12151) 2024-12-05T05:16:26,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/68c7e28a3f0d44c7b2eb6e99a347fe8c 2024-12-05T05:16:26,261 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into 8dc734efd8914e25b9a05cf71d6c163d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:26,261 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:26,261 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=12, startTime=1733375786006; duration=0sec 2024-12-05T05:16:26,261 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:26,261 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:26,278 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#33 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:26,284 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/7062e865a575461996a12679a700d39b is 50, key is test_row_0/C:col10/1733375785519/Put/seqid=0 2024-12-05T05:16:26,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/e7f0d6b38f8e4607be0338a9261e2528 is 50, key is test_row_0/B:col10/1733375785997/Put/seqid=0 2024-12-05T05:16:26,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375846262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375846268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=155, cellSize=7095, totalHeapSize=16360, min timestamp=1733375786142, max timestamp=1733375786235Num uniques 859; 2024-12-05T05:16:26,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7095, suffix heap size=117064,new segment heap size=16360 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=155 2024-12-05T05:16:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-05T05:16:26,324 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:26,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-05T05:16:26,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375846243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,330 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:26,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733375786142, max timestamp=1733375786205Num uniques 859; 2024-12-05T05:16:26,334 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T05:16:26,334 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:26,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T05:16:26,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733375786142, max timestamp=1733375786235Num uniques 859; 2024-12-05T05:16:26,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-05T05:16:26,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741870_1046 (size=12393) 2024-12-05T05:16:26,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741871_1047 (size=12151) 2024-12-05T05:16:26,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/e7f0d6b38f8e4607be0338a9261e2528 2024-12-05T05:16:26,389 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/7062e865a575461996a12679a700d39b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7062e865a575461996a12679a700d39b 2024-12-05T05:16:26,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/838caa3411a146e7ba20eaada4eab44a is 50, key is test_row_0/C:col10/1733375785997/Put/seqid=0 2024-12-05T05:16:26,408 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into 7062e865a575461996a12679a700d39b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:26,408 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:26,411 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=12, startTime=1733375786006; duration=0sec 2024-12-05T05:16:26,412 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:26,412 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T05:16:26,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741872_1048 (size=12151) 2024-12-05T05:16:26,453 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/838caa3411a146e7ba20eaada4eab44a 2024-12-05T05:16:26,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/68c7e28a3f0d44c7b2eb6e99a347fe8c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/68c7e28a3f0d44c7b2eb6e99a347fe8c 2024-12-05T05:16:26,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733375786347, max timestamp=1733375786465Num uniques 1014; 2024-12-05T05:16:26,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46438, new segment data size=12268, suffix heap size=133424,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1014, new segment cells count=268 2024-12-05T05:16:26,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/68c7e28a3f0d44c7b2eb6e99a347fe8c, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T05:16:26,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375786430, max timestamp=1733375786465Num uniques 1009; 2024-12-05T05:16:26,477 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-05T05:16:26,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/e7f0d6b38f8e4607be0338a9261e2528 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e7f0d6b38f8e4607be0338a9261e2528 2024-12-05T05:16:26,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733375786430, max timestamp=1733375786457Num uniques 1009; 2024-12-05T05:16:26,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-12-05T05:16:26,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:26,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,519 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e7f0d6b38f8e4607be0338a9261e2528, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T05:16:26,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/838caa3411a146e7ba20eaada4eab44a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/838caa3411a146e7ba20eaada4eab44a 2024-12-05T05:16:26,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/838caa3411a146e7ba20eaada4eab44a, entries=150, sequenceid=168, filesize=11.9 K 2024-12-05T05:16:26,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=124.44 KB/127422 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 542ms, sequenceid=168, compaction requested=false 2024-12-05T05:16:26,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:26,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=131.14 KB heapSize=372.55 KB 2024-12-05T05:16:26,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:26,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:26,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375846590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,606 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/adf401793eb0477cbf0372810d7f120f is 50, key is test_row_0/A:col10/1733375786564/Put/seqid=0 2024-12-05T05:16:26,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375846594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375846596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375846603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375846605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,620 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741873_1049 (size=12151) 2024-12-05T05:16:26,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:26,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.93 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/adf401793eb0477cbf0372810d7f120f 2024-12-05T05:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T05:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7f0b58eab4a14eda851d9da7acdb8d60 is 50, key is test_row_0/B:col10/1733375786564/Put/seqid=0 2024-12-05T05:16:26,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741874_1050 (size=12151) 2024-12-05T05:16:26,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7f0b58eab4a14eda851d9da7acdb8d60 2024-12-05T05:16:26,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375846713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375846713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375846724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375846724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375846725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,743 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a23cfe695ba049af810a56db6e32c3b3 is 50, key is test_row_0/C:col10/1733375786564/Put/seqid=0 2024-12-05T05:16:26,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741875_1051 (size=12151) 2024-12-05T05:16:26,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:26,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:26,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375846933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375846943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375846943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375846953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375846954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:26,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T05:16:26,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:26,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:26,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:26,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:27,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:27,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a23cfe695ba049af810a56db6e32c3b3 2024-12-05T05:16:27,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/adf401793eb0477cbf0372810d7f120f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/adf401793eb0477cbf0372810d7f120f 2024-12-05T05:16:27,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/adf401793eb0477cbf0372810d7f120f, entries=150, sequenceid=225, filesize=11.9 K 2024-12-05T05:16:27,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7f0b58eab4a14eda851d9da7acdb8d60 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0b58eab4a14eda851d9da7acdb8d60 2024-12-05T05:16:27,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0b58eab4a14eda851d9da7acdb8d60, entries=150, sequenceid=225, filesize=11.9 K 2024-12-05T05:16:27,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a23cfe695ba049af810a56db6e32c3b3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a23cfe695ba049af810a56db6e32c3b3 2024-12-05T05:16:27,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a23cfe695ba049af810a56db6e32c3b3, entries=150, sequenceid=225, filesize=11.9 K 2024-12-05T05:16:27,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~137.85 KB/141162, heapSize ~392.42 KB/401840, currentSize=40.25 KB/41220 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 671ms, sequenceid=225, compaction requested=true 2024-12-05T05:16:27,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:27,242 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:27,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:27,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:27,244 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:27,244 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:27,244 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:27,244 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:27,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:27,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:27,244 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2543ec8db930491ea275fe23a158bfda, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/68c7e28a3f0d44c7b2eb6e99a347fe8c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/adf401793eb0477cbf0372810d7f120f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=35.8 K 2024-12-05T05:16:27,245 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2543ec8db930491ea275fe23a158bfda, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375785519 2024-12-05T05:16:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:27,246 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:27,246 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:27,247 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,247 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8dc734efd8914e25b9a05cf71d6c163d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e7f0d6b38f8e4607be0338a9261e2528, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0b58eab4a14eda851d9da7acdb8d60] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=35.8 K 2024-12-05T05:16:27,247 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68c7e28a3f0d44c7b2eb6e99a347fe8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733375785849 2024-12-05T05:16:27,248 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc734efd8914e25b9a05cf71d6c163d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375785519 2024-12-05T05:16:27,249 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf401793eb0477cbf0372810d7f120f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733375786543 2024-12-05T05:16:27,250 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e7f0d6b38f8e4607be0338a9261e2528, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733375785849 2024-12-05T05:16:27,253 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0b58eab4a14eda851d9da7acdb8d60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733375786543 2024-12-05T05:16:27,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:27,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:16:27,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:27,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:27,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:27,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:27,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:27,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:27,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/e9088dbe9ffd4f7295659e69c33d9c98 is 50, key is test_row_0/A:col10/1733375787254/Put/seqid=0 2024-12-05T05:16:27,283 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:27,283 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:27,284 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/a97f895f99104ddb940e2a5bd7785b04 is 50, key is test_row_0/B:col10/1733375786564/Put/seqid=0 2024-12-05T05:16:27,284 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/6e5462fc37f540d2a6d3f7192740153f is 50, key is test_row_0/A:col10/1733375786564/Put/seqid=0 2024-12-05T05:16:27,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375847307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375847311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741876_1052 (size=12595) 2024-12-05T05:16:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375847314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375847314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375847316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,336 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/a97f895f99104ddb940e2a5bd7785b04 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a97f895f99104ddb940e2a5bd7785b04 2024-12-05T05:16:27,350 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into a97f895f99104ddb940e2a5bd7785b04(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:27,350 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:27,350 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375787244; duration=0sec 2024-12-05T05:16:27,350 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:27,351 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:27,351 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:27,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741878_1054 (size=12595) 2024-12-05T05:16:27,354 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36695 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:27,355 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:27,355 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,355 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7062e865a575461996a12679a700d39b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/838caa3411a146e7ba20eaada4eab44a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a23cfe695ba049af810a56db6e32c3b3] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=35.8 K 2024-12-05T05:16:27,356 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7062e865a575461996a12679a700d39b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375785519 2024-12-05T05:16:27,357 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 838caa3411a146e7ba20eaada4eab44a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1733375785849 2024-12-05T05:16:27,357 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a23cfe695ba049af810a56db6e32c3b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733375786543 2024-12-05T05:16:27,365 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/6e5462fc37f540d2a6d3f7192740153f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6e5462fc37f540d2a6d3f7192740153f 2024-12-05T05:16:27,375 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into 6e5462fc37f540d2a6d3f7192740153f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:27,376 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:27,376 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375787242; duration=0sec 2024-12-05T05:16:27,376 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:27,376 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:27,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741877_1053 (size=14541) 2024-12-05T05:16:27,382 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:27,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/e9088dbe9ffd4f7295659e69c33d9c98 2024-12-05T05:16:27,383 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/53182be791514586866b2c749c217ee2 is 50, key is test_row_0/C:col10/1733375786564/Put/seqid=0 2024-12-05T05:16:27,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741879_1055 (size=12595) 2024-12-05T05:16:27,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/be56ab802eea42d68efa3b198ef654cc is 50, key is test_row_0/B:col10/1733375787254/Put/seqid=0 2024-12-05T05:16:27,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375847423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375847424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:27,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741880_1056 (size=12151) 2024-12-05T05:16:27,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/be56ab802eea42d68efa3b198ef654cc 2024-12-05T05:16:27,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375847443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375847443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375847444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0f4e6491e1a646b6a6db409dede33e3e is 50, key is test_row_0/C:col10/1733375787254/Put/seqid=0 2024-12-05T05:16:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T05:16:27,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741881_1057 (size=12151) 2024-12-05T05:16:27,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375847633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,635 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375847643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375847653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375847653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375847663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:27,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,810 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/53182be791514586866b2c749c217ee2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/53182be791514586866b2c749c217ee2 2024-12-05T05:16:27,824 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into 53182be791514586866b2c749c217ee2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:27,824 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:27,824 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375787244; duration=0sec 2024-12-05T05:16:27,824 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:27,824 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:27,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0f4e6491e1a646b6a6db409dede33e3e 2024-12-05T05:16:27,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/e9088dbe9ffd4f7295659e69c33d9c98 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e9088dbe9ffd4f7295659e69c33d9c98 2024-12-05T05:16:27,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:27,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:27,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:27,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e9088dbe9ffd4f7295659e69c33d9c98, entries=200, sequenceid=237, filesize=14.2 K 2024-12-05T05:16:27,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/be56ab802eea42d68efa3b198ef654cc as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/be56ab802eea42d68efa3b198ef654cc 2024-12-05T05:16:27,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/be56ab802eea42d68efa3b198ef654cc, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T05:16:27,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0f4e6491e1a646b6a6db409dede33e3e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0f4e6491e1a646b6a6db409dede33e3e 2024-12-05T05:16:27,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375847943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:27,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0f4e6491e1a646b6a6db409dede33e3e, entries=150, sequenceid=237, filesize=11.9 K 2024-12-05T05:16:27,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 705ms, sequenceid=237, compaction requested=false 2024-12-05T05:16:27,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:27,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375787297, max timestamp=1733375787965Num uniques 859; 2024-12-05T05:16:27,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375787297, max timestamp=1733375787965Num uniques 859; 2024-12-05T05:16:27,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:27,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375787297, max timestamp=1733375787965Num uniques 859; 2024-12-05T05:16:27,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:27,969 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:27,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-05T05:16:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:27,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/3f47e642af164576bcf1fdbbcf49efdc is 50, key is test_row_0/A:col10/1733375787965/Put/seqid=0 2024-12-05T05:16:28,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741882_1058 (size=12301) 2024-12-05T05:16:28,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/3f47e642af164576bcf1fdbbcf49efdc 2024-12-05T05:16:28,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/146bfaec22f44af1bbac7053bf3c5ed3 is 50, key is test_row_0/B:col10/1733375787965/Put/seqid=0 2024-12-05T05:16:28,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:28,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375788022, max timestamp=1733375788053Num uniques 859; 2024-12-05T05:16:28,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375788022, max timestamp=1733375788053Num uniques 859; 2024-12-05T05:16:28,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:28,057 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:28,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:28,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:28,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:28,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:28,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375788039, max timestamp=1733375788053Num uniques 859; 2024-12-05T05:16:28,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:28,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741883_1059 (size=12301) 2024-12-05T05:16:28,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/146bfaec22f44af1bbac7053bf3c5ed3 2024-12-05T05:16:28,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/e364bc60a693481080930cf9142df00a is 50, key is test_row_0/C:col10/1733375787965/Put/seqid=0 2024-12-05T05:16:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375848105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375848108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,143 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375848137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375848138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741884_1060 (size=12301) 2024-12-05T05:16:28,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/e364bc60a693481080930cf9142df00a 2024-12-05T05:16:28,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/3f47e642af164576bcf1fdbbcf49efdc as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3f47e642af164576bcf1fdbbcf49efdc 2024-12-05T05:16:28,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3f47e642af164576bcf1fdbbcf49efdc, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T05:16:28,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/146bfaec22f44af1bbac7053bf3c5ed3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/146bfaec22f44af1bbac7053bf3c5ed3 2024-12-05T05:16:28,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/146bfaec22f44af1bbac7053bf3c5ed3, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T05:16:28,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/e364bc60a693481080930cf9142df00a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e364bc60a693481080930cf9142df00a 2024-12-05T05:16:28,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e364bc60a693481080930cf9142df00a, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T05:16:28,209 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~165.80 KB/169776, currentSize=131.94 KB/135110 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 231ms, sequenceid=266, compaction requested=true 2024-12-05T05:16:28,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:28,210 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:28,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-05T05:16:28,211 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,212 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:28,212 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,212 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=131.94 KB heapSize=375.22 KB 2024-12-05T05:16:28,212 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6e5462fc37f540d2a6d3f7192740153f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e9088dbe9ffd4f7295659e69c33d9c98, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3f47e642af164576bcf1fdbbcf49efdc] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=38.5 K 2024-12-05T05:16:28,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:28,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:28,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:28,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:28,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:28,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:28,213 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e5462fc37f540d2a6d3f7192740153f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733375786543 2024-12-05T05:16:28,213 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9088dbe9ffd4f7295659e69c33d9c98, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733375786585 2024-12-05T05:16:28,214 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f47e642af164576bcf1fdbbcf49efdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375787965 2024-12-05T05:16:28,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:28,228 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:28,229 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:28,229 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:28,230 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,230 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a97f895f99104ddb940e2a5bd7785b04, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/be56ab802eea42d68efa3b198ef654cc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/146bfaec22f44af1bbac7053bf3c5ed3] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.2 K 2024-12-05T05:16:28,230 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a97f895f99104ddb940e2a5bd7785b04, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733375786543 2024-12-05T05:16:28,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/fb29c053ca64464f80dfcb9c16ab6a6a is 50, key is test_row_0/A:col10/1733375788106/Put/seqid=0 2024-12-05T05:16:28,232 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting be56ab802eea42d68efa3b198ef654cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733375786585 2024-12-05T05:16:28,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:28,233 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 146bfaec22f44af1bbac7053bf3c5ed3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375787965 2024-12-05T05:16:28,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:28,239 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:28,245 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:28,246 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/10ac32f6279844e4b25ae2b1a243f942 is 50, key is test_row_0/A:col10/1733375787965/Put/seqid=0 2024-12-05T05:16:28,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:28,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375848273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375848275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375848276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375848277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,285 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#50 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:28,285 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/6b6d48ac2cb64ab3a82b181198ef1f3a is 50, key is test_row_0/B:col10/1733375787965/Put/seqid=0 2024-12-05T05:16:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741885_1061 (size=12301) 2024-12-05T05:16:28,289 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/fb29c053ca64464f80dfcb9c16ab6a6a 2024-12-05T05:16:28,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741886_1062 (size=12847) 2024-12-05T05:16:28,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/8e62faf5991548389afb22ede3579675 is 50, key is test_row_0/B:col10/1733375788106/Put/seqid=0 2024-12-05T05:16:28,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741887_1063 (size=12847) 2024-12-05T05:16:28,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741888_1064 (size=12301) 2024-12-05T05:16:28,350 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/6b6d48ac2cb64ab3a82b181198ef1f3a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6b6d48ac2cb64ab3a82b181198ef1f3a 2024-12-05T05:16:28,351 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/8e62faf5991548389afb22ede3579675 2024-12-05T05:16:28,362 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into 6b6d48ac2cb64ab3a82b181198ef1f3a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:28,362 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:28,363 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375788228; duration=0sec 2024-12-05T05:16:28,363 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:28,363 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:28,363 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:28,364 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:28,365 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:28,365 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,365 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/53182be791514586866b2c749c217ee2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0f4e6491e1a646b6a6db409dede33e3e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e364bc60a693481080930cf9142df00a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.2 K 2024-12-05T05:16:28,369 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 53182be791514586866b2c749c217ee2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1733375786543 2024-12-05T05:16:28,370 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4e6491e1a646b6a6db409dede33e3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1733375786585 2024-12-05T05:16:28,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/6579630b85dd4261ad276137d120a3e7 is 50, key is test_row_0/C:col10/1733375788106/Put/seqid=0 2024-12-05T05:16:28,373 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e364bc60a693481080930cf9142df00a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375787965 2024-12-05T05:16:28,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375848394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375848393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375848394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375848394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,412 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:28,413 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a5489d917a134d9eb9cfa25105d8f79c is 50, key is test_row_0/C:col10/1733375787965/Put/seqid=0 2024-12-05T05:16:28,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741889_1065 (size=12301) 2024-12-05T05:16:28,452 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/6579630b85dd4261ad276137d120a3e7 2024-12-05T05:16:28,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/fb29c053ca64464f80dfcb9c16ab6a6a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/fb29c053ca64464f80dfcb9c16ab6a6a 2024-12-05T05:16:28,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375848462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741890_1066 (size=12847) 2024-12-05T05:16:28,471 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/fb29c053ca64464f80dfcb9c16ab6a6a, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T05:16:28,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/8e62faf5991548389afb22ede3579675 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8e62faf5991548389afb22ede3579675 2024-12-05T05:16:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T05:16:28,483 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8e62faf5991548389afb22ede3579675, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T05:16:28,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/6579630b85dd4261ad276137d120a3e7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6579630b85dd4261ad276137d120a3e7 2024-12-05T05:16:28,497 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6579630b85dd4261ad276137d120a3e7, entries=150, sequenceid=302, filesize=12.0 K 2024-12-05T05:16:28,500 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~375.17 KB/384176, currentSize=53.67 KB/54960 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 288ms, sequenceid=302, compaction requested=false 2024-12-05T05:16:28,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:28,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-05T05:16:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-05T05:16:28,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T05:16:28,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-12-05T05:16:28,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1820 sec 2024-12-05T05:16:28,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:28,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:16:28,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:28,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:28,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:28,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:28,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:28,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:28,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/5166c5a7912b4dea915b8b1a25281c4c is 50, key is test_row_0/A:col10/1733375788618/Put/seqid=0 2024-12-05T05:16:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375848665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375848667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375848669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375848677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741891_1067 (size=12301) 2024-12-05T05:16:28,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/5166c5a7912b4dea915b8b1a25281c4c 2024-12-05T05:16:28,722 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/10ac32f6279844e4b25ae2b1a243f942 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/10ac32f6279844e4b25ae2b1a243f942 2024-12-05T05:16:28,732 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into 10ac32f6279844e4b25ae2b1a243f942(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:28,732 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:28,733 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375788210; duration=0sec 2024-12-05T05:16:28,733 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:28,733 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:28,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/6dac7f68090748c386265dea8934e34b is 50, key is test_row_0/B:col10/1733375788618/Put/seqid=0 2024-12-05T05:16:28,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741892_1068 (size=12301) 2024-12-05T05:16:28,776 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/6dac7f68090748c386265dea8934e34b 2024-12-05T05:16:28,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375848783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375848784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375848784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375848793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:28,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f117a6ea444e45b1a2fa2537ddaea3f4 is 50, key is test_row_0/C:col10/1733375788618/Put/seqid=0 2024-12-05T05:16:28,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741893_1069 (size=12301) 2024-12-05T05:16:28,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f117a6ea444e45b1a2fa2537ddaea3f4 2024-12-05T05:16:28,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/5166c5a7912b4dea915b8b1a25281c4c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5166c5a7912b4dea915b8b1a25281c4c 2024-12-05T05:16:28,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5166c5a7912b4dea915b8b1a25281c4c, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T05:16:28,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/6dac7f68090748c386265dea8934e34b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6dac7f68090748c386265dea8934e34b 2024-12-05T05:16:28,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6dac7f68090748c386265dea8934e34b, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T05:16:28,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f117a6ea444e45b1a2fa2537ddaea3f4 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f117a6ea444e45b1a2fa2537ddaea3f4 2024-12-05T05:16:28,879 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f117a6ea444e45b1a2fa2537ddaea3f4, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T05:16:28,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 260ms, sequenceid=316, compaction requested=true 2024-12-05T05:16:28,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:28,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:28,880 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:28,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:28,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:28,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:28,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:28,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T05:16:28,884 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:28,884 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:28,885 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,885 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/10ac32f6279844e4b25ae2b1a243f942, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/fb29c053ca64464f80dfcb9c16ab6a6a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5166c5a7912b4dea915b8b1a25281c4c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.6 K 2024-12-05T05:16:28,886 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10ac32f6279844e4b25ae2b1a243f942, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375787965 2024-12-05T05:16:28,889 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb29c053ca64464f80dfcb9c16ab6a6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733375788105 2024-12-05T05:16:28,889 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a5489d917a134d9eb9cfa25105d8f79c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a5489d917a134d9eb9cfa25105d8f79c 2024-12-05T05:16:28,890 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5166c5a7912b4dea915b8b1a25281c4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375788263 2024-12-05T05:16:28,904 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into a5489d917a134d9eb9cfa25105d8f79c(size=12.5 K), total size for store is 36.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:28,904 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:28,904 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375788236; duration=0sec 2024-12-05T05:16:28,905 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T05:16:28,906 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:28,906 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:28,906 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:28,909 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:28,910 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/b51cbb53ff2e484193d72aaf65f3ad19 is 50, key is test_row_0/A:col10/1733375788618/Put/seqid=0 2024-12-05T05:16:28,912 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:28,912 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:28,912 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:28,912 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6b6d48ac2cb64ab3a82b181198ef1f3a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8e62faf5991548389afb22ede3579675, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6dac7f68090748c386265dea8934e34b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.6 K 2024-12-05T05:16:28,913 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b6d48ac2cb64ab3a82b181198ef1f3a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375787965 2024-12-05T05:16:28,915 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e62faf5991548389afb22ede3579675, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733375788105 2024-12-05T05:16:28,917 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dac7f68090748c386265dea8934e34b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375788263 2024-12-05T05:16:28,938 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:28,938 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/ae82a700eb7a48b5ba507bd8df9f8938 is 50, key is test_row_0/B:col10/1733375788618/Put/seqid=0 2024-12-05T05:16:28,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741894_1070 (size=12949) 2024-12-05T05:16:28,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741895_1071 (size=12949) 2024-12-05T05:16:28,989 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/ae82a700eb7a48b5ba507bd8df9f8938 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/ae82a700eb7a48b5ba507bd8df9f8938 2024-12-05T05:16:29,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T05:16:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:29,009 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into ae82a700eb7a48b5ba507bd8df9f8938(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:29,009 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:29,009 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375788880; duration=0sec 2024-12-05T05:16:29,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:29,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:29,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:29,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:29,010 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:29,010 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:29,010 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:29,010 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:29,012 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:29,012 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:29,013 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:29,013 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a5489d917a134d9eb9cfa25105d8f79c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6579630b85dd4261ad276137d120a3e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f117a6ea444e45b1a2fa2537ddaea3f4] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.6 K 2024-12-05T05:16:29,013 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a5489d917a134d9eb9cfa25105d8f79c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375787965 2024-12-05T05:16:29,014 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6579630b85dd4261ad276137d120a3e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1733375788105 2024-12-05T05:16:29,015 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f117a6ea444e45b1a2fa2537ddaea3f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375788263 2024-12-05T05:16:29,035 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:29,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/bb65986f50574b059209feb10352e84d is 50, key is test_row_0/A:col10/1733375788999/Put/seqid=0 2024-12-05T05:16:29,035 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/93741f2fe4e6449295daf3e1987e8e9a is 50, key is test_row_0/C:col10/1733375788618/Put/seqid=0 2024-12-05T05:16:29,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375849035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375849036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375849036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375849039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,043 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741896_1072 (size=12301) 2024-12-05T05:16:29,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/bb65986f50574b059209feb10352e84d 2024-12-05T05:16:29,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/fcf2f184f6b740598975d87801950ead is 50, key is test_row_0/B:col10/1733375788999/Put/seqid=0 2024-12-05T05:16:29,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741897_1073 (size=12949) 2024-12-05T05:16:29,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741898_1074 (size=12301) 2024-12-05T05:16:29,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/fcf2f184f6b740598975d87801950ead 2024-12-05T05:16:29,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375849154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375849154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375849155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375849155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a97b5cc86c0f4cc985c013584192c276 is 50, key is test_row_0/C:col10/1733375788999/Put/seqid=0 2024-12-05T05:16:29,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741899_1075 (size=12301) 2024-12-05T05:16:29,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a97b5cc86c0f4cc985c013584192c276 2024-12-05T05:16:29,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/bb65986f50574b059209feb10352e84d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/bb65986f50574b059209feb10352e84d 2024-12-05T05:16:29,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/bb65986f50574b059209feb10352e84d, entries=150, sequenceid=340, filesize=12.0 K 2024-12-05T05:16:29,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/fcf2f184f6b740598975d87801950ead as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/fcf2f184f6b740598975d87801950ead 2024-12-05T05:16:29,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/fcf2f184f6b740598975d87801950ead, entries=150, sequenceid=340, filesize=12.0 K 2024-12-05T05:16:29,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/a97b5cc86c0f4cc985c013584192c276 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a97b5cc86c0f4cc985c013584192c276 2024-12-05T05:16:29,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a97b5cc86c0f4cc985c013584192c276, entries=150, sequenceid=340, filesize=12.0 K 2024-12-05T05:16:29,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 232ms, sequenceid=340, compaction requested=false 2024-12-05T05:16:29,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:29,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:16:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:29,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:29,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:29,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/3ee5cd63a6ec41689cc8d360b036dd86 is 50, key is test_row_0/A:col10/1733375789363/Put/seqid=0 2024-12-05T05:16:29,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741900_1076 (size=12301) 2024-12-05T05:16:29,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/3ee5cd63a6ec41689cc8d360b036dd86 2024-12-05T05:16:29,392 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/b51cbb53ff2e484193d72aaf65f3ad19 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b51cbb53ff2e484193d72aaf65f3ad19 2024-12-05T05:16:29,403 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into b51cbb53ff2e484193d72aaf65f3ad19(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:29,403 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:29,403 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375788880; duration=0sec 2024-12-05T05:16:29,404 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:29,404 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:29,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7f0e7f6053c94910ab08920214dc059c is 50, key is test_row_0/B:col10/1733375789363/Put/seqid=0 2024-12-05T05:16:29,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375849401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375849404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375849405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375849407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741901_1077 (size=12301) 2024-12-05T05:16:29,431 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7f0e7f6053c94910ab08920214dc059c 2024-12-05T05:16:29,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/fca7e9b53bb6445d819f9e0ed3efb061 is 50, key is test_row_0/C:col10/1733375789363/Put/seqid=0 2024-12-05T05:16:29,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375849483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,494 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 6ed0ce7f5160651ec0c7956ddfe8e534 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:16:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741902_1078 (size=12301) 2024-12-05T05:16:29,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/fca7e9b53bb6445d819f9e0ed3efb061 2024-12-05T05:16:29,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/3ee5cd63a6ec41689cc8d360b036dd86 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3ee5cd63a6ec41689cc8d360b036dd86 2024-12-05T05:16:29,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375849527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375849527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375849527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375849527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3ee5cd63a6ec41689cc8d360b036dd86, entries=150, sequenceid=353, filesize=12.0 K 2024-12-05T05:16:29,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/7f0e7f6053c94910ab08920214dc059c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0e7f6053c94910ab08920214dc059c 2024-12-05T05:16:29,544 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/93741f2fe4e6449295daf3e1987e8e9a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/93741f2fe4e6449295daf3e1987e8e9a 2024-12-05T05:16:29,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0e7f6053c94910ab08920214dc059c, entries=150, sequenceid=353, filesize=12.0 K 2024-12-05T05:16:29,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/fca7e9b53bb6445d819f9e0ed3efb061 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fca7e9b53bb6445d819f9e0ed3efb061 2024-12-05T05:16:29,556 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into 93741f2fe4e6449295daf3e1987e8e9a(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:29,556 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:29,557 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375788881; duration=0sec 2024-12-05T05:16:29,557 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:29,557 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:29,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fca7e9b53bb6445d819f9e0ed3efb061, entries=150, sequenceid=353, filesize=12.0 K 2024-12-05T05:16:29,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 199ms, sequenceid=353, compaction requested=true 2024-12-05T05:16:29,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:29,564 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:29,565 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:29,566 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:29,566 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:29,566 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b51cbb53ff2e484193d72aaf65f3ad19, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/bb65986f50574b059209feb10352e84d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3ee5cd63a6ec41689cc8d360b036dd86] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.7 K 2024-12-05T05:16:29,567 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b51cbb53ff2e484193d72aaf65f3ad19, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375788263 2024-12-05T05:16:29,567 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb65986f50574b059209feb10352e84d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733375788667 2024-12-05T05:16:29,568 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ee5cd63a6ec41689cc8d360b036dd86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375789032 2024-12-05T05:16:29,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:29,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:29,573 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:29,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:29,576 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:29,576 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:29,576 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:29,577 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:29,577 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/ae82a700eb7a48b5ba507bd8df9f8938, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/fcf2f184f6b740598975d87801950ead, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0e7f6053c94910ab08920214dc059c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.7 K 2024-12-05T05:16:29,578 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ae82a700eb7a48b5ba507bd8df9f8938, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375788263 2024-12-05T05:16:29,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:29,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:29,580 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting fcf2f184f6b740598975d87801950ead, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733375788667 2024-12-05T05:16:29,581 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0e7f6053c94910ab08920214dc059c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375789032 2024-12-05T05:16:29,595 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:29,596 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/edbce598c78c40fe86cce1f8a9ced33c is 50, key is test_row_0/A:col10/1733375789363/Put/seqid=0 2024-12-05T05:16:29,610 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:29,611 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/2b05f01ea76a416eb3059ea2dae84e8d is 50, key is test_row_0/B:col10/1733375789363/Put/seqid=0 2024-12-05T05:16:29,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741903_1079 (size=13051) 2024-12-05T05:16:29,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741904_1080 (size=13051) 2024-12-05T05:16:29,653 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/edbce598c78c40fe86cce1f8a9ced33c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/edbce598c78c40fe86cce1f8a9ced33c 2024-12-05T05:16:29,661 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/2b05f01ea76a416eb3059ea2dae84e8d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/2b05f01ea76a416eb3059ea2dae84e8d 2024-12-05T05:16:29,663 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into edbce598c78c40fe86cce1f8a9ced33c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:29,663 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:29,663 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375789564; duration=0sec 2024-12-05T05:16:29,664 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:29,664 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:29,664 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:29,669 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:29,669 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:29,669 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:29,669 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/93741f2fe4e6449295daf3e1987e8e9a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a97b5cc86c0f4cc985c013584192c276, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fca7e9b53bb6445d819f9e0ed3efb061] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.7 K 2024-12-05T05:16:29,670 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93741f2fe4e6449295daf3e1987e8e9a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375788263 2024-12-05T05:16:29,670 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a97b5cc86c0f4cc985c013584192c276, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1733375788667 2024-12-05T05:16:29,671 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca7e9b53bb6445d819f9e0ed3efb061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375789032 2024-12-05T05:16:29,674 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into 2b05f01ea76a416eb3059ea2dae84e8d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:29,675 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:29,675 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375789573; duration=0sec 2024-12-05T05:16:29,675 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:29,675 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:29,705 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#68 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:29,706 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/6be437933a0b4f61897348225a422a96 is 50, key is test_row_0/C:col10/1733375789363/Put/seqid=0 2024-12-05T05:16:29,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741905_1081 (size=13051) 2024-12-05T05:16:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:29,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T05:16:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375789392, max timestamp=1733375789747Num uniques 859; 2024-12-05T05:16:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375789392, max timestamp=1733375789747Num uniques 859; 2024-12-05T05:16:29,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:29,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:29,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:29,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:29,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:29,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:29,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:29,759 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/6be437933a0b4f61897348225a422a96 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6be437933a0b4f61897348225a422a96 2024-12-05T05:16:29,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/40e6143b80c0430ab5e4155c3b650fd1 is 50, key is test_row_0/A:col10/1733375789747/Put/seqid=0 2024-12-05T05:16:29,772 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into 6be437933a0b4f61897348225a422a96(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:29,772 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:29,773 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375789576; duration=0sec 2024-12-05T05:16:29,776 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:29,776 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:29,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741906_1082 (size=12301) 2024-12-05T05:16:29,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375849846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375849846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375849853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375849854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375849963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375849963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375849973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:29,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:29,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375849973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:29,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375850183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375850184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375850184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375850184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/40e6143b80c0430ab5e4155c3b650fd1 2024-12-05T05:16:30,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/697b938f94694abb994f692ac72c8ef3 is 50, key is test_row_0/B:col10/1733375789747/Put/seqid=0 2024-12-05T05:16:30,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741907_1083 (size=12301) 2024-12-05T05:16:30,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/697b938f94694abb994f692ac72c8ef3 2024-12-05T05:16:30,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0823075eb34e48e3a9357960110b3ea6 is 50, key is test_row_0/C:col10/1733375789747/Put/seqid=0 2024-12-05T05:16:30,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741908_1084 (size=12301) 2024-12-05T05:16:30,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0823075eb34e48e3a9357960110b3ea6 2024-12-05T05:16:30,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/40e6143b80c0430ab5e4155c3b650fd1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/40e6143b80c0430ab5e4155c3b650fd1 2024-12-05T05:16:30,321 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/40e6143b80c0430ab5e4155c3b650fd1, entries=150, sequenceid=379, filesize=12.0 K 2024-12-05T05:16:30,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/697b938f94694abb994f692ac72c8ef3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/697b938f94694abb994f692ac72c8ef3 2024-12-05T05:16:30,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/697b938f94694abb994f692ac72c8ef3, entries=150, sequenceid=379, filesize=12.0 K 2024-12-05T05:16:30,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/0823075eb34e48e3a9357960110b3ea6 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0823075eb34e48e3a9357960110b3ea6 2024-12-05T05:16:30,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0823075eb34e48e3a9357960110b3ea6, entries=150, sequenceid=379, filesize=12.0 K 2024-12-05T05:16:30,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~194.20 KB/198864, currentSize=107.34 KB/109920 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 594ms, sequenceid=379, compaction requested=false 2024-12-05T05:16:30,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:30,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-05T05:16:30,493 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:30,494 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-05T05:16:30,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T05:16:30,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:30,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:30,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:30,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:30,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:30,496 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:30,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T05:16:30,497 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:30,498 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:30,498 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:30,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/5dfdfd09ba784bd392ae74e67c9350c2 is 50, key is test_row_0/A:col10/1733375789849/Put/seqid=0 2024-12-05T05:16:30,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741909_1085 (size=12301) 2024-12-05T05:16:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375850509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375850511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375850511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375850511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T05:16:30,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375850622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375850622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375850623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375850623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:30,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:30,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:30,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:30,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:30,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:30,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:30,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:30,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:30,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:30,807 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:30,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:30,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:30,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T05:16:30,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375850833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375850833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375850834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:30,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375850834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:30,838 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:30,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/5dfdfd09ba784bd392ae74e67c9350c2 2024-12-05T05:16:30,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d114cf1b516b4e7d9e52800145371a51 is 50, key is test_row_0/B:col10/1733375789849/Put/seqid=0 2024-12-05T05:16:30,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741910_1086 (size=12301) 2024-12-05T05:16:30,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:30,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:30,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:30,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:31,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:31,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,118 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T05:16:31,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375851144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375851144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375851144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375851144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d114cf1b516b4e7d9e52800145371a51 2024-12-05T05:16:31,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/aef562615a8b4012b617e302d8c444f7 is 50, key is test_row_0/C:col10/1733375789849/Put/seqid=0 2024-12-05T05:16:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741911_1087 (size=12301) 2024-12-05T05:16:31,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/aef562615a8b4012b617e302d8c444f7 2024-12-05T05:16:31,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/5dfdfd09ba784bd392ae74e67c9350c2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5dfdfd09ba784bd392ae74e67c9350c2 2024-12-05T05:16:31,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5dfdfd09ba784bd392ae74e67c9350c2, entries=150, sequenceid=400, filesize=12.0 K 2024-12-05T05:16:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d114cf1b516b4e7d9e52800145371a51 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d114cf1b516b4e7d9e52800145371a51 2024-12-05T05:16:31,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:31,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,440 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d114cf1b516b4e7d9e52800145371a51, entries=150, sequenceid=400, filesize=12.0 K 2024-12-05T05:16:31,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/aef562615a8b4012b617e302d8c444f7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/aef562615a8b4012b617e302d8c444f7 2024-12-05T05:16:31,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/aef562615a8b4012b617e302d8c444f7, entries=150, sequenceid=400, filesize=12.0 K 2024-12-05T05:16:31,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 959ms, sequenceid=400, compaction requested=true 2024-12-05T05:16:31,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:31,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:31,454 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:31,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:31,455 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:31,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:31,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:31,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:31,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:31,456 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:31,456 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:31,457 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,457 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/edbce598c78c40fe86cce1f8a9ced33c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/40e6143b80c0430ab5e4155c3b650fd1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5dfdfd09ba784bd392ae74e67c9350c2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.8 K 2024-12-05T05:16:31,457 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:31,457 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:31,457 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,457 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/2b05f01ea76a416eb3059ea2dae84e8d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/697b938f94694abb994f692ac72c8ef3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d114cf1b516b4e7d9e52800145371a51] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.8 K 2024-12-05T05:16:31,458 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting edbce598c78c40fe86cce1f8a9ced33c, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375789032 2024-12-05T05:16:31,459 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b05f01ea76a416eb3059ea2dae84e8d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375789032 2024-12-05T05:16:31,459 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40e6143b80c0430ab5e4155c3b650fd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733375789404 2024-12-05T05:16:31,459 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 697b938f94694abb994f692ac72c8ef3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733375789404 2024-12-05T05:16:31,460 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dfdfd09ba784bd392ae74e67c9350c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375789842 2024-12-05T05:16:31,460 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d114cf1b516b4e7d9e52800145371a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375789842 2024-12-05T05:16:31,482 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:31,483 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/a0e4f960b71d47c19bb2cfbb787ed8ce is 50, key is test_row_0/B:col10/1733375789849/Put/seqid=0 2024-12-05T05:16:31,489 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#76 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:31,490 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/023fb87c90a64015be9428e999f4a325 is 50, key is test_row_0/A:col10/1733375789849/Put/seqid=0 2024-12-05T05:16:31,505 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:16:31,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:31,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:31,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:31,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:31,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:31,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:31,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:31,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741913_1089 (size=13153) 2024-12-05T05:16:31,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741912_1088 (size=13153) 2024-12-05T05:16:31,532 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/023fb87c90a64015be9428e999f4a325 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/023fb87c90a64015be9428e999f4a325 2024-12-05T05:16:31,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/f41922fbe90c47ecaeaf7b46b87228b1 is 50, key is test_row_0/A:col10/1733375791504/Put/seqid=0 2024-12-05T05:16:31,545 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into 023fb87c90a64015be9428e999f4a325(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:31,546 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:31,546 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375791454; duration=0sec 2024-12-05T05:16:31,546 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:31,546 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:31,546 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:31,548 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/a0e4f960b71d47c19bb2cfbb787ed8ce as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a0e4f960b71d47c19bb2cfbb787ed8ce 2024-12-05T05:16:31,562 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into a0e4f960b71d47c19bb2cfbb787ed8ce(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:31,562 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:31,562 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375791455; duration=0sec 2024-12-05T05:16:31,562 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:31,562 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:31,565 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:31,565 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:31,565 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,565 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6be437933a0b4f61897348225a422a96, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0823075eb34e48e3a9357960110b3ea6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/aef562615a8b4012b617e302d8c444f7] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.8 K 2024-12-05T05:16:31,566 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be437933a0b4f61897348225a422a96, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375789032 2024-12-05T05:16:31,566 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0823075eb34e48e3a9357960110b3ea6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1733375789404 2024-12-05T05:16:31,567 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting aef562615a8b4012b617e302d8c444f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375789842 2024-12-05T05:16:31,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741914_1090 (size=14741) 2024-12-05T05:16:31,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/f41922fbe90c47ecaeaf7b46b87228b1 2024-12-05T05:16:31,588 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:31,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:31,589 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f66a5219d0be42be8e45d2e4ce365a19 is 50, key is test_row_0/C:col10/1733375789849/Put/seqid=0 2024-12-05T05:16:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d1f6f733978a4250ae640a112e8ca672 is 50, key is test_row_0/B:col10/1733375791504/Put/seqid=0 2024-12-05T05:16:31,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741915_1091 (size=13153) 2024-12-05T05:16:31,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T05:16:31,642 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f66a5219d0be42be8e45d2e4ce365a19 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f66a5219d0be42be8e45d2e4ce365a19 2024-12-05T05:16:31,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375851646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741916_1092 (size=12301) 2024-12-05T05:16:31,655 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into f66a5219d0be42be8e45d2e4ce365a19(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:31,655 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:31,656 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375791455; duration=0sec 2024-12-05T05:16:31,656 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:31,656 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:31,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375851653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375851653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375851653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375851653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:31,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:31,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375851752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,755 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:31,896 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:31,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:31,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:31,897 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:31,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:31,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375851962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:31,963 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:32,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:32,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d1f6f733978a4250ae640a112e8ca672 2024-12-05T05:16:32,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/25c5a3e2176446238da4d4fc9dd22a39 is 50, key is test_row_0/C:col10/1733375791504/Put/seqid=0 2024-12-05T05:16:32,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741917_1093 (size=12301) 2024-12-05T05:16:32,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375852273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/25c5a3e2176446238da4d4fc9dd22a39 2024-12-05T05:16:32,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/f41922fbe90c47ecaeaf7b46b87228b1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f41922fbe90c47ecaeaf7b46b87228b1 2024-12-05T05:16:32,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f41922fbe90c47ecaeaf7b46b87228b1, entries=200, sequenceid=413, filesize=14.4 K 2024-12-05T05:16:32,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d1f6f733978a4250ae640a112e8ca672 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d1f6f733978a4250ae640a112e8ca672 2024-12-05T05:16:32,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d1f6f733978a4250ae640a112e8ca672, entries=150, sequenceid=413, filesize=12.0 K 2024-12-05T05:16:32,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/25c5a3e2176446238da4d4fc9dd22a39 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/25c5a3e2176446238da4d4fc9dd22a39 2024-12-05T05:16:32,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/25c5a3e2176446238da4d4fc9dd22a39, entries=150, sequenceid=413, filesize=12.0 K 2024-12-05T05:16:32,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1014ms, sequenceid=413, compaction requested=false 2024-12-05T05:16:32,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T05:16:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:32,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T05:16:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:32,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:32,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:32,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:32,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/4f09281cc464481b919bc19577be262e is 50, key is test_row_0/A:col10/1733375791591/Put/seqid=0 2024-12-05T05:16:32,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375852688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375852689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375852697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,703 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375852698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741918_1094 (size=14741) 2024-12-05T05:16:32,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375852793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375852803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375852804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375852813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:32,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375852814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:32,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:32,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:32,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:32,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375853014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375853014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375853023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375853023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,025 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/4f09281cc464481b919bc19577be262e 2024-12-05T05:16:33,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/35e713ba37b6411fb11884f10e212acd is 50, key is test_row_0/B:col10/1733375791591/Put/seqid=0 2024-12-05T05:16:33,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741919_1095 (size=12301) 2024-12-05T05:16:33,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:33,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:33,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:33,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375853323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375853324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375853332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375853333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:33,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:33,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/35e713ba37b6411fb11884f10e212acd 2024-12-05T05:16:33,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/d5decb9305a94b20aa61bb4ae7c00203 is 50, key is test_row_0/C:col10/1733375791591/Put/seqid=0 2024-12-05T05:16:33,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741920_1096 (size=12301) 2024-12-05T05:16:33,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:33,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:33,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:33,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:33,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375853803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,804 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 6ed0ce7f5160651ec0c7956ddfe8e534 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375853832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375853833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375853843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375853843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:33,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:33,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:33,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:33,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:33,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/d5decb9305a94b20aa61bb4ae7c00203 2024-12-05T05:16:33,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/4f09281cc464481b919bc19577be262e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/4f09281cc464481b919bc19577be262e 2024-12-05T05:16:33,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/4f09281cc464481b919bc19577be262e, entries=200, sequenceid=436, filesize=14.4 K 2024-12-05T05:16:33,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/35e713ba37b6411fb11884f10e212acd as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/35e713ba37b6411fb11884f10e212acd 2024-12-05T05:16:33,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/35e713ba37b6411fb11884f10e212acd, entries=150, sequenceid=436, filesize=12.0 K 2024-12-05T05:16:33,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/d5decb9305a94b20aa61bb4ae7c00203 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/d5decb9305a94b20aa61bb4ae7c00203 2024-12-05T05:16:33,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/d5decb9305a94b20aa61bb4ae7c00203, entries=150, sequenceid=436, filesize=12.0 K 2024-12-05T05:16:33,978 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1312ms, sequenceid=436, compaction requested=true 2024-12-05T05:16:33,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:33,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:33,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:33,978 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:33,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:33,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:33,978 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:33,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:33,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:33,979 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:33,979 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:33,979 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,979 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/023fb87c90a64015be9428e999f4a325, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f41922fbe90c47ecaeaf7b46b87228b1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/4f09281cc464481b919bc19577be262e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=41.6 K 2024-12-05T05:16:33,980 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:33,980 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:33,980 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:33,981 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a0e4f960b71d47c19bb2cfbb787ed8ce, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d1f6f733978a4250ae640a112e8ca672, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/35e713ba37b6411fb11884f10e212acd] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.9 K 2024-12-05T05:16:33,981 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 023fb87c90a64015be9428e999f4a325, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375789842 2024-12-05T05:16:33,981 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f41922fbe90c47ecaeaf7b46b87228b1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733375790507 2024-12-05T05:16:33,981 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a0e4f960b71d47c19bb2cfbb787ed8ce, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375789842 2024-12-05T05:16:33,982 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d1f6f733978a4250ae640a112e8ca672, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733375790508 2024-12-05T05:16:33,982 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f09281cc464481b919bc19577be262e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733375791591 2024-12-05T05:16:33,983 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 35e713ba37b6411fb11884f10e212acd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733375791591 2024-12-05T05:16:34,005 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:34,006 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/631e429c63fa41cd99b35647b666df5b is 50, key is test_row_0/A:col10/1733375791591/Put/seqid=0 2024-12-05T05:16:34,022 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#85 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:34,024 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/482baf9cc23642efac6370a908290b80 is 50, key is test_row_0/B:col10/1733375791591/Put/seqid=0 2024-12-05T05:16:34,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741921_1097 (size=13255) 2024-12-05T05:16:34,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741922_1098 (size=13255) 2024-12-05T05:16:34,044 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/631e429c63fa41cd99b35647b666df5b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/631e429c63fa41cd99b35647b666df5b 2024-12-05T05:16:34,055 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into 631e429c63fa41cd99b35647b666df5b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:34,055 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:34,055 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375793978; duration=0sec 2024-12-05T05:16:34,055 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:34,055 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:34,055 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:34,057 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:34,057 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:34,057 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:34,058 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f66a5219d0be42be8e45d2e4ce365a19, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/25c5a3e2176446238da4d4fc9dd22a39, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/d5decb9305a94b20aa61bb4ae7c00203] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=36.9 K 2024-12-05T05:16:34,058 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f66a5219d0be42be8e45d2e4ce365a19, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375789842 2024-12-05T05:16:34,059 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25c5a3e2176446238da4d4fc9dd22a39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733375790508 2024-12-05T05:16:34,059 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5decb9305a94b20aa61bb4ae7c00203, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733375791591 2024-12-05T05:16:34,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-05T05:16:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:34,064 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T05:16:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:34,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:34,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:34,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:34,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:34,074 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#86 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:34,075 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f404df228e2a497396257f5f367912b1 is 50, key is test_row_0/C:col10/1733375791591/Put/seqid=0 2024-12-05T05:16:34,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/86995d1a9d93459b96dd4ed63b625061 is 50, key is test_row_0/A:col10/1733375792689/Put/seqid=0 2024-12-05T05:16:34,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741923_1099 (size=13255) 2024-12-05T05:16:34,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741924_1100 (size=12301) 2024-12-05T05:16:34,145 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/86995d1a9d93459b96dd4ed63b625061 2024-12-05T05:16:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d8d9e75452284be380d53d554ba25698 is 50, key is test_row_0/B:col10/1733375792689/Put/seqid=0 2024-12-05T05:16:34,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741925_1101 (size=12301) 2024-12-05T05:16:34,174 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d8d9e75452284be380d53d554ba25698 2024-12-05T05:16:34,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/c24a5366a6c443cdbae4ed8316cb56e7 is 50, key is test_row_0/C:col10/1733375792689/Put/seqid=0 2024-12-05T05:16:34,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741926_1102 (size=12301) 2024-12-05T05:16:34,449 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/482baf9cc23642efac6370a908290b80 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/482baf9cc23642efac6370a908290b80 2024-12-05T05:16:34,460 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into 482baf9cc23642efac6370a908290b80(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:34,460 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:34,460 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375793978; duration=0sec 2024-12-05T05:16:34,463 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:34,463 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:34,533 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/f404df228e2a497396257f5f367912b1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f404df228e2a497396257f5f367912b1 2024-12-05T05:16:34,558 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into f404df228e2a497396257f5f367912b1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:34,558 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:34,558 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375793978; duration=0sec 2024-12-05T05:16:34,558 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:34,558 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:34,602 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/c24a5366a6c443cdbae4ed8316cb56e7 2024-12-05T05:16:34,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/86995d1a9d93459b96dd4ed63b625061 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/86995d1a9d93459b96dd4ed63b625061 2024-12-05T05:16:34,624 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/86995d1a9d93459b96dd4ed63b625061, entries=150, sequenceid=449, filesize=12.0 K 2024-12-05T05:16:34,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/d8d9e75452284be380d53d554ba25698 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d8d9e75452284be380d53d554ba25698 2024-12-05T05:16:34,631 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d8d9e75452284be380d53d554ba25698, entries=150, sequenceid=449, filesize=12.0 K 2024-12-05T05:16:34,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/c24a5366a6c443cdbae4ed8316cb56e7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c24a5366a6c443cdbae4ed8316cb56e7 2024-12-05T05:16:34,640 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c24a5366a6c443cdbae4ed8316cb56e7, entries=150, sequenceid=449, filesize=12.0 K 2024-12-05T05:16:34,642 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 578ms, sequenceid=449, compaction requested=false 2024-12-05T05:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:34,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-05T05:16:34,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-05T05:16:34,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-05T05:16:34,648 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.1470 sec 2024-12-05T05:16:34,650 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 4.1540 sec 2024-12-05T05:16:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-05T05:16:34,664 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:34,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-05T05:16:34,668 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T05:16:34,671 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:34,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T05:16:34,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-05T05:16:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:34,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-05T05:16:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-05T05:16:34,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T05:16:34,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-12-05T05:16:34,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 165 msec 2024-12-05T05:16:34,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:16:34,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:34,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:34,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:34,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:34,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:34,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:34,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/76b6fd523a2b4d76aab523ec7c05261c is 50, key is test_row_0/A:col10/1733375794843/Put/seqid=0 2024-12-05T05:16:34,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733375794878, max timestamp=1733375794890Num uniques 859; 2024-12-05T05:16:34,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733375794878, max timestamp=1733375794890Num uniques 859; 2024-12-05T05:16:34,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733375794878, max timestamp=1733375794890Num uniques 859; 2024-12-05T05:16:34,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T05:16:34,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T05:16:34,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T05:16:34,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741927_1103 (size=12301) 2024-12-05T05:16:34,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/76b6fd523a2b4d76aab523ec7c05261c 2024-12-05T05:16:34,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/384d5fad364c45cf8e3e6bf8d4a4fb9d is 50, key is test_row_0/B:col10/1733375794843/Put/seqid=0 2024-12-05T05:16:34,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741928_1104 (size=12301) 2024-12-05T05:16:34,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375854939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:34,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375854948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:34,954 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:34,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375854949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:34,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375854949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-05T05:16:34,983 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:34,983 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-05T05:16:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T05:16:34,987 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:34,988 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:34,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:35,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375855053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375855062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375855063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375855063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T05:16:35,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375855273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375855273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375855273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375855273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:35,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T05:16:35,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/384d5fad364c45cf8e3e6bf8d4a4fb9d 2024-12-05T05:16:35,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/e2a674d63bec41f7a8727a9621078096 is 50, key is test_row_0/C:col10/1733375794843/Put/seqid=0 2024-12-05T05:16:35,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741929_1105 (size=12301) 2024-12-05T05:16:35,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:35,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:35,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375855583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375855583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375855583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375855583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:35,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:35,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T05:16:35,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/e2a674d63bec41f7a8727a9621078096 2024-12-05T05:16:35,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/76b6fd523a2b4d76aab523ec7c05261c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/76b6fd523a2b4d76aab523ec7c05261c 2024-12-05T05:16:35,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/76b6fd523a2b4d76aab523ec7c05261c, entries=150, sequenceid=462, filesize=12.0 K 2024-12-05T05:16:35,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/384d5fad364c45cf8e3e6bf8d4a4fb9d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/384d5fad364c45cf8e3e6bf8d4a4fb9d 2024-12-05T05:16:35,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/384d5fad364c45cf8e3e6bf8d4a4fb9d, entries=150, sequenceid=462, filesize=12.0 K 2024-12-05T05:16:35,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/e2a674d63bec41f7a8727a9621078096 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e2a674d63bec41f7a8727a9621078096 2024-12-05T05:16:35,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e2a674d63bec41f7a8727a9621078096, entries=150, sequenceid=462, filesize=12.0 K 2024-12-05T05:16:35,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 923ms, sequenceid=462, compaction requested=true 2024-12-05T05:16:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:35,778 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:35,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:35,779 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:35,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:35,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ed0ce7f5160651ec0c7956ddfe8e534:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:35,779 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:35,780 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:35,780 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:35,780 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/A is initiating minor compaction (all files) 2024-12-05T05:16:35,780 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/B is initiating minor compaction (all files) 2024-12-05T05:16:35,780 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/A in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,780 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/B in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,780 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/482baf9cc23642efac6370a908290b80, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d8d9e75452284be380d53d554ba25698, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/384d5fad364c45cf8e3e6bf8d4a4fb9d] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=37.0 K 2024-12-05T05:16:35,780 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/631e429c63fa41cd99b35647b666df5b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/86995d1a9d93459b96dd4ed63b625061, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/76b6fd523a2b4d76aab523ec7c05261c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=37.0 K 2024-12-05T05:16:35,781 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 482baf9cc23642efac6370a908290b80, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733375791591 2024-12-05T05:16:35,781 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631e429c63fa41cd99b35647b666df5b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733375791591 2024-12-05T05:16:35,781 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d8d9e75452284be380d53d554ba25698, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733375792687 2024-12-05T05:16:35,781 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86995d1a9d93459b96dd4ed63b625061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733375792687 2024-12-05T05:16:35,781 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 384d5fad364c45cf8e3e6bf8d4a4fb9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733375794843 2024-12-05T05:16:35,781 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b6fd523a2b4d76aab523ec7c05261c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733375794843 2024-12-05T05:16:35,789 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#A#compaction#93 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:35,790 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/f7538ea4ba4d400eafbf2d4779dbf6fd is 50, key is test_row_0/A:col10/1733375794843/Put/seqid=0 2024-12-05T05:16:35,792 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:35,792 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/e6a6fa8f13bb4693a3814f32dff0ad22 is 50, key is test_row_0/B:col10/1733375794843/Put/seqid=0 2024-12-05T05:16:35,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741930_1106 (size=13357) 2024-12-05T05:16:35,804 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/f7538ea4ba4d400eafbf2d4779dbf6fd as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f7538ea4ba4d400eafbf2d4779dbf6fd 2024-12-05T05:16:35,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741931_1107 (size=13357) 2024-12-05T05:16:35,811 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/A of 6ed0ce7f5160651ec0c7956ddfe8e534 into f7538ea4ba4d400eafbf2d4779dbf6fd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:35,811 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:35,811 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/A, priority=13, startTime=1733375795778; duration=0sec 2024-12-05T05:16:35,812 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:35,812 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:A 2024-12-05T05:16:35,812 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:35,814 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:35,814 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 6ed0ce7f5160651ec0c7956ddfe8e534/C is initiating minor compaction (all files) 2024-12-05T05:16:35,814 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ed0ce7f5160651ec0c7956ddfe8e534/C in TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,814 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f404df228e2a497396257f5f367912b1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c24a5366a6c443cdbae4ed8316cb56e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e2a674d63bec41f7a8727a9621078096] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp, totalSize=37.0 K 2024-12-05T05:16:35,815 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f404df228e2a497396257f5f367912b1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1733375791591 2024-12-05T05:16:35,815 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/e6a6fa8f13bb4693a3814f32dff0ad22 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e6a6fa8f13bb4693a3814f32dff0ad22 2024-12-05T05:16:35,815 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c24a5366a6c443cdbae4ed8316cb56e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1733375792687 2024-12-05T05:16:35,816 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2a674d63bec41f7a8727a9621078096, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1733375794843 2024-12-05T05:16:35,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375794938, max timestamp=1733375795817Num uniques 1118; 2024-12-05T05:16:35,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-05T05:16:35,822 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/B of 6ed0ce7f5160651ec0c7956ddfe8e534 into e6a6fa8f13bb4693a3814f32dff0ad22(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:35,822 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:35,822 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/B, priority=13, startTime=1733375795778; duration=0sec 2024-12-05T05:16:35,823 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:35,823 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:B 2024-12-05T05:16:35,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:35,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=111.82 KB heapSize=312.52 KB 2024-12-05T05:16:35,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:35,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:35,823 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:35,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375794938, max timestamp=1733375795817Num uniques 1118; 2024-12-05T05:16:35,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-05T05:16:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T05:16:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:35,825 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:35,829 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ed0ce7f5160651ec0c7956ddfe8e534#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:35,830 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/e3b1050cecec4762a8ea20d58e8c1f94 is 50, key is test_row_0/C:col10/1733375794843/Put/seqid=0 2024-12-05T05:16:35,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/8ddb2f01e3ef4953bf8098f99b870232 is 50, key is test_row_0/A:col10/1733375794947/Put/seqid=0 2024-12-05T05:16:35,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741932_1108 (size=13357) 2024-12-05T05:16:35,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741933_1109 (size=14741) 2024-12-05T05:16:35,855 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/e3b1050cecec4762a8ea20d58e8c1f94 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e3b1050cecec4762a8ea20d58e8c1f94 2024-12-05T05:16:35,862 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ed0ce7f5160651ec0c7956ddfe8e534/C of 6ed0ce7f5160651ec0c7956ddfe8e534 into e3b1050cecec4762a8ea20d58e8c1f94(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:35,862 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:35,862 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., storeName=6ed0ce7f5160651ec0c7956ddfe8e534/C, priority=13, startTime=1733375795779; duration=0sec 2024-12-05T05:16:35,862 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:35,862 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ed0ce7f5160651ec0c7956ddfe8e534:C 2024-12-05T05:16:35,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375855885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,888 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:35,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:35,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:35,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:35,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:35,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:35,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375855993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:35,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:36,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1733375856093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:36,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38718 deadline: 1733375856093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:36,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38678 deadline: 1733375856093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:36,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1733375856104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T05:16:36,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375856203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:36,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:36,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:36,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/8ddb2f01e3ef4953bf8098f99b870232 2024-12-05T05:16:36,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/74269c0bb2ea450e9b0eda09805a0cc6 is 50, key is test_row_0/B:col10/1733375794947/Put/seqid=0 2024-12-05T05:16:36,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741934_1110 (size=12301) 2024-12-05T05:16:36,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:36,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:36,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38666 deadline: 1733375856523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:16:36,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:36,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/74269c0bb2ea450e9b0eda09805a0cc6 2024-12-05T05:16:36,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/fc8dc8c976e34107950cd6e61935e260 is 50, key is test_row_0/C:col10/1733375794947/Put/seqid=0 2024-12-05T05:16:36,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741935_1111 (size=12301) 2024-12-05T05:16:36,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/fc8dc8c976e34107950cd6e61935e260 2024-12-05T05:16:36,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/8ddb2f01e3ef4953bf8098f99b870232 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8ddb2f01e3ef4953bf8098f99b870232 2024-12-05T05:16:36,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8ddb2f01e3ef4953bf8098f99b870232, entries=200, sequenceid=502, filesize=14.4 K 2024-12-05T05:16:36,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/74269c0bb2ea450e9b0eda09805a0cc6 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/74269c0bb2ea450e9b0eda09805a0cc6 2024-12-05T05:16:36,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/74269c0bb2ea450e9b0eda09805a0cc6, entries=150, sequenceid=502, filesize=12.0 K 2024-12-05T05:16:36,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/fc8dc8c976e34107950cd6e61935e260 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fc8dc8c976e34107950cd6e61935e260 2024-12-05T05:16:36,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fc8dc8c976e34107950cd6e61935e260, entries=150, sequenceid=502, filesize=12.0 K 2024-12-05T05:16:36,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~188.91 KB/193440, currentSize=114.05 KB/116790 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 964ms, sequenceid=502, compaction requested=false 2024-12-05T05:16:36,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:36,799 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-05T05:16:36,799 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:36,800 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-05T05:16:36,800 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:36,801 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-05T05:16:36,801 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:36,805 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:36,805 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:36,805 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:36,805 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-05T05:16:36,805 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:36,805 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:36,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:36,806 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:36,806 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:36,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:36,806 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:36,806 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:36,807 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:36,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:36,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-05T05:16:36,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:36,835 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T05:16:36,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:36,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:36,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:36,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/89e95f5096d444d49767a758d03a9007 is 50, key is test_row_0/A:col10/1733375795883/Put/seqid=0 2024-12-05T05:16:36,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741936_1112 (size=9857) 2024-12-05T05:16:37,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:37,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. as already flushing 2024-12-05T05:16:37,035 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-05T05:16:37,035 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:37,035 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,035 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:37,105 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-05T05:16:37,105 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-05T05:16:37,105 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-05T05:16:37,105 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:37,105 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:37,105 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:37,105 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,105 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,105 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,105 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,105 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,105 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:37,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:37,105 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:37,124 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-05T05:16:37,124 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:16:37,124 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,124 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:37,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:37,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T05:16:37,245 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/89e95f5096d444d49767a758d03a9007 2024-12-05T05:16:37,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/712c5e458a9b49f78c69255a62c82377 is 50, key is test_row_0/B:col10/1733375795883/Put/seqid=0 2024-12-05T05:16:37,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741937_1113 (size=9857) 2024-12-05T05:16:37,661 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/712c5e458a9b49f78c69255a62c82377 2024-12-05T05:16:37,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/3ceed23eb1db4db696ffe41c13f9960c is 50, key is test_row_0/C:col10/1733375795883/Put/seqid=0 2024-12-05T05:16:37,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741938_1114 (size=9857) 2024-12-05T05:16:38,073 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/3ceed23eb1db4db696ffe41c13f9960c 2024-12-05T05:16:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/89e95f5096d444d49767a758d03a9007 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/89e95f5096d444d49767a758d03a9007 2024-12-05T05:16:38,088 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/89e95f5096d444d49767a758d03a9007, entries=100, sequenceid=523, filesize=9.6 K 2024-12-05T05:16:38,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/712c5e458a9b49f78c69255a62c82377 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/712c5e458a9b49f78c69255a62c82377 2024-12-05T05:16:38,093 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/712c5e458a9b49f78c69255a62c82377, entries=100, sequenceid=523, filesize=9.6 K 2024-12-05T05:16:38,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/3ceed23eb1db4db696ffe41c13f9960c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/3ceed23eb1db4db696ffe41c13f9960c 2024-12-05T05:16:38,098 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/3ceed23eb1db4db696ffe41c13f9960c, entries=100, sequenceid=523, filesize=9.6 K 2024-12-05T05:16:38,099 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=33.54 KB/34350 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1264ms, sequenceid=523, compaction requested=true 2024-12-05T05:16:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6ed0ce7f5160651ec0c7956ddfe8e534: 2024-12-05T05:16:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:38,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-05T05:16:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-05T05:16:38,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T05:16:38,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1130 sec 2024-12-05T05:16:38,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 3.1200 sec 2024-12-05T05:16:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-05T05:16:39,153 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2388 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2339 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1135 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3403 rows 2024-12-05T05:16:39,153 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1125 2024-12-05T05:16:39,154 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3373 rows 2024-12-05T05:16:39,154 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T05:16:39,154 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:16:39,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:39,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:39,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:16:39,159 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T05:16:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T05:16:39,167 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375799167"}]},"ts":"1733375799167"} 2024-12-05T05:16:39,169 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T05:16:39,169 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T05:16:39,171 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:16:39,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, UNASSIGN}] 2024-12-05T05:16:39,175 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, UNASSIGN 2024-12-05T05:16:39,177 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=6ed0ce7f5160651ec0c7956ddfe8e534, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:39,179 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, UNASSIGN because future has completed 2024-12-05T05:16:39,179 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:16:39,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:16:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T05:16:39,336 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:39,336 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:16:39,336 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 6ed0ce7f5160651ec0c7956ddfe8e534, disabling compactions & flushes 2024-12-05T05:16:39,337 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. after waiting 0 ms 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:39,337 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 6ed0ce7f5160651ec0c7956ddfe8e534 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=A 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=B 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ed0ce7f5160651ec0c7956ddfe8e534, store=C 2024-12-05T05:16:39,337 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:39,342 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/1baabbba4926477c803b823b7c15e3cb is 50, key is test_row_0/A:col10/1733375797103/Put/seqid=0 2024-12-05T05:16:39,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741939_1115 (size=9857) 2024-12-05T05:16:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T05:16:39,517 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T05:16:39,747 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/1baabbba4926477c803b823b7c15e3cb 2024-12-05T05:16:39,759 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/65641f6ef5134162a7e486e3a2799e9e is 50, key is test_row_0/B:col10/1733375797103/Put/seqid=0 2024-12-05T05:16:39,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741940_1116 (size=9857) 2024-12-05T05:16:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T05:16:40,165 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/65641f6ef5134162a7e486e3a2799e9e 2024-12-05T05:16:40,174 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/6553b0cbd2ff40e8a0ef6412acf7e91e is 50, key is test_row_0/C:col10/1733375797103/Put/seqid=0 2024-12-05T05:16:40,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741941_1117 (size=9857) 2024-12-05T05:16:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T05:16:40,580 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/6553b0cbd2ff40e8a0ef6412acf7e91e 2024-12-05T05:16:40,588 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/A/1baabbba4926477c803b823b7c15e3cb as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/1baabbba4926477c803b823b7c15e3cb 2024-12-05T05:16:40,594 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/1baabbba4926477c803b823b7c15e3cb, entries=100, sequenceid=531, filesize=9.6 K 2024-12-05T05:16:40,595 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/B/65641f6ef5134162a7e486e3a2799e9e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/65641f6ef5134162a7e486e3a2799e9e 2024-12-05T05:16:40,600 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/65641f6ef5134162a7e486e3a2799e9e, entries=100, sequenceid=531, filesize=9.6 K 2024-12-05T05:16:40,600 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/.tmp/C/6553b0cbd2ff40e8a0ef6412acf7e91e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6553b0cbd2ff40e8a0ef6412acf7e91e 2024-12-05T05:16:40,605 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6553b0cbd2ff40e8a0ef6412acf7e91e, entries=100, sequenceid=531, filesize=9.6 K 2024-12-05T05:16:40,606 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1269ms, sequenceid=531, compaction requested=true 2024-12-05T05:16:40,607 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8fa89a669e48410298b67065581e8463, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2f97f673babb4df49d53b49eeba6410d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e25d5714eacd46899d97de8489a784d2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6bd2f8c76ade4be2810f474674ea1310, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/7f2481826ea34bc59f184a136a7b0675, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/30265cbd621649958a0b7efdd589c620, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/982b7d538d9244a383eab773087a09c3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/946352fb63484168bff636c2437ad7cd, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b57f78a2e7ee4cb785c12bc3b23af83d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/497b4682c46541409e59fb980dfb8465, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2543ec8db930491ea275fe23a158bfda, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/68c7e28a3f0d44c7b2eb6e99a347fe8c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6e5462fc37f540d2a6d3f7192740153f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/adf401793eb0477cbf0372810d7f120f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e9088dbe9ffd4f7295659e69c33d9c98, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/10ac32f6279844e4b25ae2b1a243f942, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3f47e642af164576bcf1fdbbcf49efdc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/fb29c053ca64464f80dfcb9c16ab6a6a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b51cbb53ff2e484193d72aaf65f3ad19, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5166c5a7912b4dea915b8b1a25281c4c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/bb65986f50574b059209feb10352e84d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/edbce598c78c40fe86cce1f8a9ced33c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3ee5cd63a6ec41689cc8d360b036dd86, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/40e6143b80c0430ab5e4155c3b650fd1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/023fb87c90a64015be9428e999f4a325, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5dfdfd09ba784bd392ae74e67c9350c2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f41922fbe90c47ecaeaf7b46b87228b1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/4f09281cc464481b919bc19577be262e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/631e429c63fa41cd99b35647b666df5b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/86995d1a9d93459b96dd4ed63b625061, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/76b6fd523a2b4d76aab523ec7c05261c] to archive 2024-12-05T05:16:40,611 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:16:40,616 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8fa89a669e48410298b67065581e8463 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8fa89a669e48410298b67065581e8463 2024-12-05T05:16:40,618 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2f97f673babb4df49d53b49eeba6410d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2f97f673babb4df49d53b49eeba6410d 2024-12-05T05:16:40,620 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e25d5714eacd46899d97de8489a784d2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e25d5714eacd46899d97de8489a784d2 2024-12-05T05:16:40,621 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6bd2f8c76ade4be2810f474674ea1310 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6bd2f8c76ade4be2810f474674ea1310 2024-12-05T05:16:40,622 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/7f2481826ea34bc59f184a136a7b0675 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/7f2481826ea34bc59f184a136a7b0675 2024-12-05T05:16:40,624 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/30265cbd621649958a0b7efdd589c620 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/30265cbd621649958a0b7efdd589c620 2024-12-05T05:16:40,625 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/982b7d538d9244a383eab773087a09c3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/982b7d538d9244a383eab773087a09c3 2024-12-05T05:16:40,627 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/946352fb63484168bff636c2437ad7cd to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/946352fb63484168bff636c2437ad7cd 2024-12-05T05:16:40,628 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b57f78a2e7ee4cb785c12bc3b23af83d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b57f78a2e7ee4cb785c12bc3b23af83d 2024-12-05T05:16:40,630 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/497b4682c46541409e59fb980dfb8465 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/497b4682c46541409e59fb980dfb8465 2024-12-05T05:16:40,632 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2543ec8db930491ea275fe23a158bfda to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/2543ec8db930491ea275fe23a158bfda 2024-12-05T05:16:40,634 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/68c7e28a3f0d44c7b2eb6e99a347fe8c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/68c7e28a3f0d44c7b2eb6e99a347fe8c 2024-12-05T05:16:40,635 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6e5462fc37f540d2a6d3f7192740153f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/6e5462fc37f540d2a6d3f7192740153f 2024-12-05T05:16:40,641 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/adf401793eb0477cbf0372810d7f120f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/adf401793eb0477cbf0372810d7f120f 2024-12-05T05:16:40,642 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e9088dbe9ffd4f7295659e69c33d9c98 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/e9088dbe9ffd4f7295659e69c33d9c98 2024-12-05T05:16:40,643 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/10ac32f6279844e4b25ae2b1a243f942 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/10ac32f6279844e4b25ae2b1a243f942 2024-12-05T05:16:40,645 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3f47e642af164576bcf1fdbbcf49efdc to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3f47e642af164576bcf1fdbbcf49efdc 2024-12-05T05:16:40,646 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/fb29c053ca64464f80dfcb9c16ab6a6a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/fb29c053ca64464f80dfcb9c16ab6a6a 2024-12-05T05:16:40,648 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b51cbb53ff2e484193d72aaf65f3ad19 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/b51cbb53ff2e484193d72aaf65f3ad19 2024-12-05T05:16:40,649 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5166c5a7912b4dea915b8b1a25281c4c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5166c5a7912b4dea915b8b1a25281c4c 2024-12-05T05:16:40,650 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/bb65986f50574b059209feb10352e84d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/bb65986f50574b059209feb10352e84d 2024-12-05T05:16:40,651 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/edbce598c78c40fe86cce1f8a9ced33c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/edbce598c78c40fe86cce1f8a9ced33c 2024-12-05T05:16:40,652 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3ee5cd63a6ec41689cc8d360b036dd86 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/3ee5cd63a6ec41689cc8d360b036dd86 2024-12-05T05:16:40,654 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/40e6143b80c0430ab5e4155c3b650fd1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/40e6143b80c0430ab5e4155c3b650fd1 2024-12-05T05:16:40,655 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/023fb87c90a64015be9428e999f4a325 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/023fb87c90a64015be9428e999f4a325 2024-12-05T05:16:40,656 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5dfdfd09ba784bd392ae74e67c9350c2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/5dfdfd09ba784bd392ae74e67c9350c2 2024-12-05T05:16:40,658 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f41922fbe90c47ecaeaf7b46b87228b1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f41922fbe90c47ecaeaf7b46b87228b1 2024-12-05T05:16:40,659 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/4f09281cc464481b919bc19577be262e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/4f09281cc464481b919bc19577be262e 2024-12-05T05:16:40,660 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/631e429c63fa41cd99b35647b666df5b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/631e429c63fa41cd99b35647b666df5b 2024-12-05T05:16:40,662 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/86995d1a9d93459b96dd4ed63b625061 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/86995d1a9d93459b96dd4ed63b625061 2024-12-05T05:16:40,663 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/76b6fd523a2b4d76aab523ec7c05261c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/76b6fd523a2b4d76aab523ec7c05261c 2024-12-05T05:16:40,674 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9ec8ee513f2a41babcef1b4b8da4f3b2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7ae9eb650ff847bbb4d096d06e0dd465, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/5618428a273043f884493f65d4363ecf, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/95dc830dd04c4e63b0caf0940256d6ab, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/f0485d2dc6d44c13a2422077dc1c0283, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8838db0fc69d4c3b8ddf16e97b68763a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/44ad257f09154aa49e60d245d86f3766, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/dcf82ae66fb24ccb9576f33e31e9a1df, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9884f65f1eeb4e4b8dc1ad7f027a7697, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8dc734efd8914e25b9a05cf71d6c163d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/899d832415594f3cabd4ff034825d625, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e7f0d6b38f8e4607be0338a9261e2528, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a97f895f99104ddb940e2a5bd7785b04, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0b58eab4a14eda851d9da7acdb8d60, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/be56ab802eea42d68efa3b198ef654cc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6b6d48ac2cb64ab3a82b181198ef1f3a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/146bfaec22f44af1bbac7053bf3c5ed3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8e62faf5991548389afb22ede3579675, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/ae82a700eb7a48b5ba507bd8df9f8938, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6dac7f68090748c386265dea8934e34b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/fcf2f184f6b740598975d87801950ead, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/2b05f01ea76a416eb3059ea2dae84e8d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0e7f6053c94910ab08920214dc059c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/697b938f94694abb994f692ac72c8ef3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a0e4f960b71d47c19bb2cfbb787ed8ce, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d114cf1b516b4e7d9e52800145371a51, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d1f6f733978a4250ae640a112e8ca672, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/482baf9cc23642efac6370a908290b80, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/35e713ba37b6411fb11884f10e212acd, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d8d9e75452284be380d53d554ba25698, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/384d5fad364c45cf8e3e6bf8d4a4fb9d] to archive 2024-12-05T05:16:40,676 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:16:40,678 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9ec8ee513f2a41babcef1b4b8da4f3b2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9ec8ee513f2a41babcef1b4b8da4f3b2 2024-12-05T05:16:40,679 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7ae9eb650ff847bbb4d096d06e0dd465 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7ae9eb650ff847bbb4d096d06e0dd465 2024-12-05T05:16:40,680 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/5618428a273043f884493f65d4363ecf to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/5618428a273043f884493f65d4363ecf 2024-12-05T05:16:40,682 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/95dc830dd04c4e63b0caf0940256d6ab to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/95dc830dd04c4e63b0caf0940256d6ab 2024-12-05T05:16:40,683 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/f0485d2dc6d44c13a2422077dc1c0283 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/f0485d2dc6d44c13a2422077dc1c0283 2024-12-05T05:16:40,684 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8838db0fc69d4c3b8ddf16e97b68763a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8838db0fc69d4c3b8ddf16e97b68763a 2024-12-05T05:16:40,685 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/44ad257f09154aa49e60d245d86f3766 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/44ad257f09154aa49e60d245d86f3766 2024-12-05T05:16:40,686 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/dcf82ae66fb24ccb9576f33e31e9a1df to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/dcf82ae66fb24ccb9576f33e31e9a1df 2024-12-05T05:16:40,687 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9884f65f1eeb4e4b8dc1ad7f027a7697 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/9884f65f1eeb4e4b8dc1ad7f027a7697 2024-12-05T05:16:40,688 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8dc734efd8914e25b9a05cf71d6c163d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8dc734efd8914e25b9a05cf71d6c163d 2024-12-05T05:16:40,690 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/899d832415594f3cabd4ff034825d625 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/899d832415594f3cabd4ff034825d625 2024-12-05T05:16:40,691 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e7f0d6b38f8e4607be0338a9261e2528 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e7f0d6b38f8e4607be0338a9261e2528 2024-12-05T05:16:40,692 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a97f895f99104ddb940e2a5bd7785b04 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a97f895f99104ddb940e2a5bd7785b04 2024-12-05T05:16:40,693 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0b58eab4a14eda851d9da7acdb8d60 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0b58eab4a14eda851d9da7acdb8d60 2024-12-05T05:16:40,694 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/be56ab802eea42d68efa3b198ef654cc to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/be56ab802eea42d68efa3b198ef654cc 2024-12-05T05:16:40,695 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6b6d48ac2cb64ab3a82b181198ef1f3a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6b6d48ac2cb64ab3a82b181198ef1f3a 2024-12-05T05:16:40,696 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/146bfaec22f44af1bbac7053bf3c5ed3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/146bfaec22f44af1bbac7053bf3c5ed3 2024-12-05T05:16:40,697 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8e62faf5991548389afb22ede3579675 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/8e62faf5991548389afb22ede3579675 2024-12-05T05:16:40,698 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/ae82a700eb7a48b5ba507bd8df9f8938 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/ae82a700eb7a48b5ba507bd8df9f8938 2024-12-05T05:16:40,699 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6dac7f68090748c386265dea8934e34b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/6dac7f68090748c386265dea8934e34b 2024-12-05T05:16:40,700 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/fcf2f184f6b740598975d87801950ead to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/fcf2f184f6b740598975d87801950ead 2024-12-05T05:16:40,701 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/2b05f01ea76a416eb3059ea2dae84e8d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/2b05f01ea76a416eb3059ea2dae84e8d 2024-12-05T05:16:40,702 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0e7f6053c94910ab08920214dc059c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/7f0e7f6053c94910ab08920214dc059c 2024-12-05T05:16:40,703 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/697b938f94694abb994f692ac72c8ef3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/697b938f94694abb994f692ac72c8ef3 2024-12-05T05:16:40,704 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a0e4f960b71d47c19bb2cfbb787ed8ce to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/a0e4f960b71d47c19bb2cfbb787ed8ce 2024-12-05T05:16:40,705 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d114cf1b516b4e7d9e52800145371a51 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d114cf1b516b4e7d9e52800145371a51 2024-12-05T05:16:40,706 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d1f6f733978a4250ae640a112e8ca672 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d1f6f733978a4250ae640a112e8ca672 2024-12-05T05:16:40,707 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/482baf9cc23642efac6370a908290b80 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/482baf9cc23642efac6370a908290b80 2024-12-05T05:16:40,708 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/35e713ba37b6411fb11884f10e212acd to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/35e713ba37b6411fb11884f10e212acd 2024-12-05T05:16:40,709 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d8d9e75452284be380d53d554ba25698 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/d8d9e75452284be380d53d554ba25698 2024-12-05T05:16:40,710 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/384d5fad364c45cf8e3e6bf8d4a4fb9d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/384d5fad364c45cf8e3e6bf8d4a4fb9d 2024-12-05T05:16:40,711 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5141392d483c416bb5f25bdf5ecd6be1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/776525c7039941b1aac36c40f6db432e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f3f8d8e303b84111958c898240f71642, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0b40ba9c375f47e697419a2a286881f9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5b03cedfe9cc4bee9d997d858d172c37, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/72c55c57017f46fc933e09a3897f4d4e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c32e7ea3f28d4658af0dec9af2f8125f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a802a65425464bc78e3080ffbf6a958d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0619bb7e48ad4622acfdfca45723ea3f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7062e865a575461996a12679a700d39b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7c828096b2154e219493c73389db2044, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/838caa3411a146e7ba20eaada4eab44a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/53182be791514586866b2c749c217ee2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a23cfe695ba049af810a56db6e32c3b3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0f4e6491e1a646b6a6db409dede33e3e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a5489d917a134d9eb9cfa25105d8f79c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e364bc60a693481080930cf9142df00a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6579630b85dd4261ad276137d120a3e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/93741f2fe4e6449295daf3e1987e8e9a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f117a6ea444e45b1a2fa2537ddaea3f4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a97b5cc86c0f4cc985c013584192c276, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6be437933a0b4f61897348225a422a96, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fca7e9b53bb6445d819f9e0ed3efb061, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0823075eb34e48e3a9357960110b3ea6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f66a5219d0be42be8e45d2e4ce365a19, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/aef562615a8b4012b617e302d8c444f7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/25c5a3e2176446238da4d4fc9dd22a39, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f404df228e2a497396257f5f367912b1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/d5decb9305a94b20aa61bb4ae7c00203, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c24a5366a6c443cdbae4ed8316cb56e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e2a674d63bec41f7a8727a9621078096] to archive 2024-12-05T05:16:40,712 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:16:40,714 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5141392d483c416bb5f25bdf5ecd6be1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5141392d483c416bb5f25bdf5ecd6be1 2024-12-05T05:16:40,715 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/776525c7039941b1aac36c40f6db432e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/776525c7039941b1aac36c40f6db432e 2024-12-05T05:16:40,716 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f3f8d8e303b84111958c898240f71642 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f3f8d8e303b84111958c898240f71642 2024-12-05T05:16:40,717 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0b40ba9c375f47e697419a2a286881f9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0b40ba9c375f47e697419a2a286881f9 2024-12-05T05:16:40,718 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5b03cedfe9cc4bee9d997d858d172c37 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/5b03cedfe9cc4bee9d997d858d172c37 2024-12-05T05:16:40,719 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/72c55c57017f46fc933e09a3897f4d4e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/72c55c57017f46fc933e09a3897f4d4e 2024-12-05T05:16:40,720 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c32e7ea3f28d4658af0dec9af2f8125f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c32e7ea3f28d4658af0dec9af2f8125f 2024-12-05T05:16:40,721 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a802a65425464bc78e3080ffbf6a958d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a802a65425464bc78e3080ffbf6a958d 2024-12-05T05:16:40,722 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0619bb7e48ad4622acfdfca45723ea3f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0619bb7e48ad4622acfdfca45723ea3f 2024-12-05T05:16:40,723 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7062e865a575461996a12679a700d39b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7062e865a575461996a12679a700d39b 2024-12-05T05:16:40,724 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7c828096b2154e219493c73389db2044 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/7c828096b2154e219493c73389db2044 2024-12-05T05:16:40,725 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/838caa3411a146e7ba20eaada4eab44a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/838caa3411a146e7ba20eaada4eab44a 2024-12-05T05:16:40,726 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/53182be791514586866b2c749c217ee2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/53182be791514586866b2c749c217ee2 2024-12-05T05:16:40,727 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a23cfe695ba049af810a56db6e32c3b3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a23cfe695ba049af810a56db6e32c3b3 2024-12-05T05:16:40,728 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0f4e6491e1a646b6a6db409dede33e3e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0f4e6491e1a646b6a6db409dede33e3e 2024-12-05T05:16:40,729 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a5489d917a134d9eb9cfa25105d8f79c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a5489d917a134d9eb9cfa25105d8f79c 2024-12-05T05:16:40,730 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e364bc60a693481080930cf9142df00a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e364bc60a693481080930cf9142df00a 2024-12-05T05:16:40,731 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6579630b85dd4261ad276137d120a3e7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6579630b85dd4261ad276137d120a3e7 2024-12-05T05:16:40,732 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/93741f2fe4e6449295daf3e1987e8e9a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/93741f2fe4e6449295daf3e1987e8e9a 2024-12-05T05:16:40,734 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f117a6ea444e45b1a2fa2537ddaea3f4 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f117a6ea444e45b1a2fa2537ddaea3f4 2024-12-05T05:16:40,735 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a97b5cc86c0f4cc985c013584192c276 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/a97b5cc86c0f4cc985c013584192c276 2024-12-05T05:16:40,737 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6be437933a0b4f61897348225a422a96 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6be437933a0b4f61897348225a422a96 2024-12-05T05:16:40,738 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fca7e9b53bb6445d819f9e0ed3efb061 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fca7e9b53bb6445d819f9e0ed3efb061 2024-12-05T05:16:40,742 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0823075eb34e48e3a9357960110b3ea6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/0823075eb34e48e3a9357960110b3ea6 2024-12-05T05:16:40,743 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f66a5219d0be42be8e45d2e4ce365a19 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f66a5219d0be42be8e45d2e4ce365a19 2024-12-05T05:16:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/aef562615a8b4012b617e302d8c444f7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/aef562615a8b4012b617e302d8c444f7 2024-12-05T05:16:40,747 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/25c5a3e2176446238da4d4fc9dd22a39 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/25c5a3e2176446238da4d4fc9dd22a39 2024-12-05T05:16:40,749 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f404df228e2a497396257f5f367912b1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/f404df228e2a497396257f5f367912b1 2024-12-05T05:16:40,750 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/d5decb9305a94b20aa61bb4ae7c00203 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/d5decb9305a94b20aa61bb4ae7c00203 2024-12-05T05:16:40,752 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c24a5366a6c443cdbae4ed8316cb56e7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/c24a5366a6c443cdbae4ed8316cb56e7 2024-12-05T05:16:40,753 DEBUG [StoreCloser-TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e2a674d63bec41f7a8727a9621078096 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e2a674d63bec41f7a8727a9621078096 2024-12-05T05:16:40,757 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/recovered.edits/534.seqid, newMaxSeqId=534, maxSeqId=1 2024-12-05T05:16:40,758 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. 2024-12-05T05:16:40,758 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 6ed0ce7f5160651ec0c7956ddfe8e534: Waiting for close lock at 1733375799336Running coprocessor pre-close hooks at 1733375799336Disabling compacts and flushes for region at 1733375799336Disabling writes for close at 1733375799337 (+1 ms)Obtaining lock to block concurrent updates at 1733375799337Preparing flush snapshotting stores in 6ed0ce7f5160651ec0c7956ddfe8e534 at 1733375799337Finished memstore snapshotting TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733375799337Flushing stores of TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534. at 1733375799338 (+1 ms)Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/A: creating writer at 1733375799338Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/A: appending metadata at 1733375799341 (+3 ms)Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/A: closing flushed file at 1733375799341Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/B: creating writer at 1733375799755 (+414 ms)Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/B: appending metadata at 1733375799759 (+4 ms)Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/B: closing flushed file at 1733375799759Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/C: creating writer at 1733375800170 (+411 ms)Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/C: appending metadata at 1733375800174 (+4 ms)Flushing 6ed0ce7f5160651ec0c7956ddfe8e534/C: closing flushed file at 1733375800174Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45762223: reopening flushed file at 1733375800587 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@234fe47: reopening flushed file at 1733375800594 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65113316: reopening flushed file at 1733375800600 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6ed0ce7f5160651ec0c7956ddfe8e534 in 1269ms, sequenceid=531, compaction requested=true at 1733375800606 (+6 ms)Writing region close event to WAL at 1733375800754 (+148 ms)Running coprocessor post-close hooks at 1733375800758 (+4 ms)Closed at 1733375800758 2024-12-05T05:16:40,760 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:40,760 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=6ed0ce7f5160651ec0c7956ddfe8e534, regionState=CLOSED 2024-12-05T05:16:40,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:16:40,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-05T05:16:40,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ed0ce7f5160651ec0c7956ddfe8e534, server=8edcaf2ecc84,36233,1733375771401 in 1.5840 sec 2024-12-05T05:16:40,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-05T05:16:40,766 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ed0ce7f5160651ec0c7956ddfe8e534, UNASSIGN in 1.5900 sec 2024-12-05T05:16:40,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-05T05:16:40,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5970 sec 2024-12-05T05:16:40,770 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375800769"}]},"ts":"1733375800769"} 2024-12-05T05:16:40,771 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T05:16:40,771 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T05:16:40,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6110 sec 2024-12-05T05:16:40,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:16:41,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-05T05:16:41,314 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:41,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T05:16:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:41,326 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:41,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T05:16:41,328 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:41,334 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:41,340 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/recovered.edits] 2024-12-05T05:16:41,344 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/1baabbba4926477c803b823b7c15e3cb to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/1baabbba4926477c803b823b7c15e3cb 2024-12-05T05:16:41,345 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/89e95f5096d444d49767a758d03a9007 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/89e95f5096d444d49767a758d03a9007 2024-12-05T05:16:41,346 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8ddb2f01e3ef4953bf8098f99b870232 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/8ddb2f01e3ef4953bf8098f99b870232 2024-12-05T05:16:41,348 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f7538ea4ba4d400eafbf2d4779dbf6fd to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/A/f7538ea4ba4d400eafbf2d4779dbf6fd 2024-12-05T05:16:41,351 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/65641f6ef5134162a7e486e3a2799e9e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/65641f6ef5134162a7e486e3a2799e9e 2024-12-05T05:16:41,352 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/712c5e458a9b49f78c69255a62c82377 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/712c5e458a9b49f78c69255a62c82377 2024-12-05T05:16:41,353 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/74269c0bb2ea450e9b0eda09805a0cc6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/74269c0bb2ea450e9b0eda09805a0cc6 2024-12-05T05:16:41,355 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e6a6fa8f13bb4693a3814f32dff0ad22 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/B/e6a6fa8f13bb4693a3814f32dff0ad22 2024-12-05T05:16:41,359 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/3ceed23eb1db4db696ffe41c13f9960c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/3ceed23eb1db4db696ffe41c13f9960c 2024-12-05T05:16:41,365 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6553b0cbd2ff40e8a0ef6412acf7e91e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/6553b0cbd2ff40e8a0ef6412acf7e91e 2024-12-05T05:16:41,368 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e3b1050cecec4762a8ea20d58e8c1f94 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/e3b1050cecec4762a8ea20d58e8c1f94 2024-12-05T05:16:41,374 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fc8dc8c976e34107950cd6e61935e260 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/C/fc8dc8c976e34107950cd6e61935e260 2024-12-05T05:16:41,379 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/recovered.edits/534.seqid to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534/recovered.edits/534.seqid 2024-12-05T05:16:41,379 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/6ed0ce7f5160651ec0c7956ddfe8e534 2024-12-05T05:16:41,379 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T05:16:41,386 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:41,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-05T05:16:41,405 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T05:16:41,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T05:16:41,452 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T05:16:41,454 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:41,455 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T05:16:41,455 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733375801455"}]},"ts":"9223372036854775807"} 2024-12-05T05:16:41,461 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T05:16:41,461 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6ed0ce7f5160651ec0c7956ddfe8e534, NAME => 'TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T05:16:41,462 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T05:16:41,462 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733375801462"}]},"ts":"9223372036854775807"} 2024-12-05T05:16:41,467 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T05:16:41,468 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:41,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 148 msec 2024-12-05T05:16:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-05T05:16:41,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T05:16:41,644 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:41,665 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_195128769_22 at /127.0.0.1:48628 [Waiting for operation #266] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_195128769_22 at /127.0.0.1:35966 [Waiting for operation #125] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050451830_22 at /127.0.0.1:41588 [Waiting for operation #278] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;8edcaf2ecc84:36233-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=846 (was 683) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3122 (was 3121) - AvailableMemoryMB LEAK? - 2024-12-05T05:16:41,688 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=846, ProcessCount=11, AvailableMemoryMB=3118 2024-12-05T05:16:41,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:16:41,720 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:16:41,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:41,723 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T05:16:41,724 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:41,724 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-12-05T05:16:41,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T05:16:41,725 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T05:16:41,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741942_1118 (size=1061) 2024-12-05T05:16:41,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T05:16:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T05:16:42,136 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 2024-12-05T05:16:42,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741943_1119 (size=53) 2024-12-05T05:16:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T05:16:42,543 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:42,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 09d09c462e929eda7f34ea9ccdd56981, disabling compactions & flushes 2024-12-05T05:16:42,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:42,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:42,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. after waiting 0 ms 2024-12-05T05:16:42,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:42,544 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:42,544 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 09d09c462e929eda7f34ea9ccdd56981: Waiting for close lock at 1733375802544Disabling compacts and flushes for region at 1733375802544Disabling writes for close at 1733375802544Writing region close event to WAL at 1733375802544Closed at 1733375802544 2024-12-05T05:16:42,545 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T05:16:42,545 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733375802545"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733375802545"}]},"ts":"1733375802545"} 2024-12-05T05:16:42,547 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T05:16:42,549 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T05:16:42,549 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375802549"}]},"ts":"1733375802549"} 2024-12-05T05:16:42,551 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T05:16:42,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, ASSIGN}] 2024-12-05T05:16:42,552 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, ASSIGN 2024-12-05T05:16:42,553 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, ASSIGN; state=OFFLINE, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=false 2024-12-05T05:16:42,704 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=09d09c462e929eda7f34ea9ccdd56981, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:42,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, ASSIGN because future has completed 2024-12-05T05:16:42,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:16:42,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T05:16:42,873 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:42,873 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:16:42,873 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,873 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:42,874 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,874 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,875 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,877 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:42,878 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d09c462e929eda7f34ea9ccdd56981 columnFamilyName A 2024-12-05T05:16:42,878 DEBUG [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:42,878 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(327): Store=09d09c462e929eda7f34ea9ccdd56981/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:42,879 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,880 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:42,880 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d09c462e929eda7f34ea9ccdd56981 columnFamilyName B 2024-12-05T05:16:42,880 DEBUG [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:42,881 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(327): Store=09d09c462e929eda7f34ea9ccdd56981/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:42,881 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,882 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:42,882 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d09c462e929eda7f34ea9ccdd56981 columnFamilyName C 2024-12-05T05:16:42,882 DEBUG [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:42,883 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(327): Store=09d09c462e929eda7f34ea9ccdd56981/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:42,883 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:42,883 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,884 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,884 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,885 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,885 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,886 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:16:42,887 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,890 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T05:16:42,890 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 09d09c462e929eda7f34ea9ccdd56981; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65747491, jitterRate=-0.020286038517951965}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:16:42,890 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:42,891 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 09d09c462e929eda7f34ea9ccdd56981: Running coprocessor pre-open hook at 1733375802874Writing region info on filesystem at 1733375802874Initializing all the Stores at 1733375802875 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375802875Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375802875Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375802875Cleaning up temporary data from old regions at 1733375802885 (+10 ms)Running coprocessor post-open hooks at 1733375802890 (+5 ms)Region opened successfully at 1733375802891 (+1 ms) 2024-12-05T05:16:42,896 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., pid=28, masterSystemTime=1733375802868 2024-12-05T05:16:42,899 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:42,899 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:42,899 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=09d09c462e929eda7f34ea9ccdd56981, regionState=OPEN, openSeqNum=2, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:42,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:16:42,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-05T05:16:42,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 in 189 msec 2024-12-05T05:16:42,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-12-05T05:16:42,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, ASSIGN in 356 msec 2024-12-05T05:16:42,914 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T05:16:42,914 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375802914"}]},"ts":"1733375802914"} 2024-12-05T05:16:42,916 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T05:16:42,917 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T05:16:42,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1970 sec 2024-12-05T05:16:43,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-12-05T05:16:43,873 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:43,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b4e2bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:43,874 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:43,874 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:43,876 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@595cff53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:43,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:43,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:43,878 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:43,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d5b06de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:43,879 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:43,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:43,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:43,884 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T05:16:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 8edcaf2ecc84,46695,1733375770505 2024-12-05T05:16:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1b7a241d 2024-12-05T05:16:43,885 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T05:16:43,887 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T05:16:43,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:16:43,893 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:16:43,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T05:16:43,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741944_1120 (size=1097) 2024-12-05T05:16:43,931 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T05:16:43,931 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T05:16:43,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:16:43,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, REOPEN/MOVE}] 2024-12-05T05:16:43,950 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, REOPEN/MOVE 2024-12-05T05:16:43,951 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=09d09c462e929eda7f34ea9ccdd56981, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:43,954 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, REOPEN/MOVE because future has completed 2024-12-05T05:16:43,965 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:16:43,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:16:44,119 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,119 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:16:44,120 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 09d09c462e929eda7f34ea9ccdd56981, disabling compactions & flushes 2024-12-05T05:16:44,120 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,120 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,120 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. after waiting 0 ms 2024-12-05T05:16:44,120 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,125 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T05:16:44,126 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,126 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 09d09c462e929eda7f34ea9ccdd56981: Waiting for close lock at 1733375804120Running coprocessor pre-close hooks at 1733375804120Disabling compacts and flushes for region at 1733375804120Disabling writes for close at 1733375804120Writing region close event to WAL at 1733375804120Running coprocessor post-close hooks at 1733375804126 (+6 ms)Closed at 1733375804126 2024-12-05T05:16:44,126 WARN [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 09d09c462e929eda7f34ea9ccdd56981 to self. 2024-12-05T05:16:44,129 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,130 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=09d09c462e929eda7f34ea9ccdd56981, regionState=CLOSED 2024-12-05T05:16:44,131 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:16:44,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-05T05:16:44,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 in 167 msec 2024-12-05T05:16:44,135 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, REOPEN/MOVE; state=CLOSED, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=true 2024-12-05T05:16:44,286 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=09d09c462e929eda7f34ea9ccdd56981, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,288 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, REOPEN/MOVE because future has completed 2024-12-05T05:16:44,289 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:16:44,443 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,444 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:16:44,444 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,444 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:16:44,444 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,444 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,447 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,448 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:44,452 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d09c462e929eda7f34ea9ccdd56981 columnFamilyName A 2024-12-05T05:16:44,454 DEBUG [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:44,455 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(327): Store=09d09c462e929eda7f34ea9ccdd56981/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:44,455 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,456 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:44,456 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d09c462e929eda7f34ea9ccdd56981 columnFamilyName B 2024-12-05T05:16:44,456 DEBUG [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:44,457 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(327): Store=09d09c462e929eda7f34ea9ccdd56981/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:44,457 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,458 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:16:44,458 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09d09c462e929eda7f34ea9ccdd56981 columnFamilyName C 2024-12-05T05:16:44,458 DEBUG [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:44,458 INFO [StoreOpener-09d09c462e929eda7f34ea9ccdd56981-1 {}] regionserver.HStore(327): Store=09d09c462e929eda7f34ea9ccdd56981/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:16:44,459 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,459 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,460 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,461 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,462 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,462 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,463 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:16:44,464 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,465 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 09d09c462e929eda7f34ea9ccdd56981; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73808871, jitterRate=0.09983788430690765}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:16:44,465 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,466 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 09d09c462e929eda7f34ea9ccdd56981: Running coprocessor pre-open hook at 1733375804444Writing region info on filesystem at 1733375804445 (+1 ms)Initializing all the Stores at 1733375804445Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375804446 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375804446Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375804446Cleaning up temporary data from old regions at 1733375804462 (+16 ms)Running coprocessor post-open hooks at 1733375804465 (+3 ms)Region opened successfully at 1733375804466 (+1 ms) 2024-12-05T05:16:44,467 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., pid=33, masterSystemTime=1733375804440 2024-12-05T05:16:44,469 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,469 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,470 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=09d09c462e929eda7f34ea9ccdd56981, regionState=OPEN, openSeqNum=5, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:16:44,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-12-05T05:16:44,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 in 184 msec 2024-12-05T05:16:44,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-05T05:16:44,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, REOPEN/MOVE in 527 msec 2024-12-05T05:16:44,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T05:16:44,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 543 msec 2024-12-05T05:16:44,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 585 msec 2024-12-05T05:16:44,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-05T05:16:44,485 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:44,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37870970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,487 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70128bcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,487 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,489 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65fc31d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454c5b1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,491 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@214e0be8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,492 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,493 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51653fe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72aba2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,496 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60b6c7d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,497 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,498 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c0706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bbd945b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,499 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,500 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,500 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17990eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,501 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,502 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ede757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d26dd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,504 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546ef564, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,505 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@545d3fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@182dc420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,507 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@134ec61a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,509 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@779600d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4500591c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,511 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ba2433b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,511 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,512 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a0d289b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e853ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,514 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e2d70d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,515 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7461d5cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6230e7af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:16:44,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:16:44,517 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:16:44,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:16:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:16:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5be74e29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:16:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:16:44,518 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:16:44,519 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:16:44,520 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a91d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:16:44,520 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,520 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,520 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,520 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,520 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,520 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,521 DEBUG [Thread-615 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,522 DEBUG [Thread-615 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,522 DEBUG [Thread-613 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,522 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,524 DEBUG [Thread-613 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,529 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,531 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,531 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:44,532 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,532 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:16:44,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:16:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-12-05T05:16:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:16:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,535 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:44,535 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T05:16:44,536 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:44,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:44,537 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,538 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:16:44,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,545 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:16:44,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:16:44,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:44,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:44,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:44,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:44,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:44,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:44,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T05:16:44,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375804625, max timestamp=1733375804664Num uniques 859; 2024-12-05T05:16:44,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375804625, max timestamp=1733375804664Num uniques 859; 2024-12-05T05:16:44,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:44,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:44,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375804625, max timestamp=1733375804664Num uniques 859; 2024-12-05T05:16:44,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:44,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205643b588c1d49454a9e075fa92c245495_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375804553/Put/seqid=0 2024-12-05T05:16:44,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:44,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:44,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:44,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:44,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741945_1121 (size=17034) 2024-12-05T05:16:44,720 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:44,731 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205643b588c1d49454a9e075fa92c245495_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205643b588c1d49454a9e075fa92c245495_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:44,735 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/39aa317f5e894268927e92c8ebcf791a, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:44,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/39aa317f5e894268927e92c8ebcf791a is 175, key is test_row_0/A:col10/1733375804553/Put/seqid=0 2024-12-05T05:16:44,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375864749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375864753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375864755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375864758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375864762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741946_1122 (size=48135) 2024-12-05T05:16:44,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T05:16:44,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:44,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:44,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375864873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375864873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,877 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,885 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375864883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375864883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:44,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:44,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375864883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:44,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:45,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375865084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375865084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,092 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375865094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375865094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375865095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T05:16:45,176 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:45,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:45,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,177 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,212 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/39aa317f5e894268927e92c8ebcf791a 2024-12-05T05:16:45,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/a45a95332aa246f6a5629527bf0e746a is 50, key is test_row_0/B:col10/1733375804553/Put/seqid=0 2024-12-05T05:16:45,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741947_1123 (size=9657) 2024-12-05T05:16:45,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:45,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:45,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375865406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375865408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,419 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375865424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375865424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,444 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375865424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:45,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:45,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:45,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T05:16:45,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/a45a95332aa246f6a5629527bf0e746a 2024-12-05T05:16:45,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/567090afe8514cd5b2486eac8b6e792d is 50, key is test_row_0/C:col10/1733375804553/Put/seqid=0 2024-12-05T05:16:45,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741948_1124 (size=9657) 2024-12-05T05:16:45,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375865923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375865923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375865955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375865963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375865963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:45,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:45,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:46,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:46,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:46,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:46,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/567090afe8514cd5b2486eac8b6e792d 2024-12-05T05:16:46,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/39aa317f5e894268927e92c8ebcf791a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/39aa317f5e894268927e92c8ebcf791a 2024-12-05T05:16:46,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:46,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:46,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:46,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/39aa317f5e894268927e92c8ebcf791a, entries=250, sequenceid=15, filesize=47.0 K 2024-12-05T05:16:46,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/a45a95332aa246f6a5629527bf0e746a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a45a95332aa246f6a5629527bf0e746a 2024-12-05T05:16:46,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a45a95332aa246f6a5629527bf0e746a, entries=100, sequenceid=15, filesize=9.4 K 2024-12-05T05:16:46,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/567090afe8514cd5b2486eac8b6e792d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/567090afe8514cd5b2486eac8b6e792d 2024-12-05T05:16:46,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/567090afe8514cd5b2486eac8b6e792d, entries=100, sequenceid=15, filesize=9.4 K 2024-12-05T05:16:46,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 09d09c462e929eda7f34ea9ccdd56981 in 1775ms, sequenceid=15, compaction requested=false 2024-12-05T05:16:46,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:46,386 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T05:16:46,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-12-05T05:16:46,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:46,464 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-05T05:16:46,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:46,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:46,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:46,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:46,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:46,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:46,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053a59ff94915949ec97bc0c4147826177_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375804751/Put/seqid=0 2024-12-05T05:16:46,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741949_1125 (size=12154) 2024-12-05T05:16:46,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:46,585 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053a59ff94915949ec97bc0c4147826177_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a59ff94915949ec97bc0c4147826177_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:46,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/05f63486d53e4af98cbf94a777da4c7d, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:46,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/05f63486d53e4af98cbf94a777da4c7d is 175, key is test_row_0/A:col10/1733375804751/Put/seqid=0 2024-12-05T05:16:46,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741950_1126 (size=30955) 2024-12-05T05:16:46,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T05:16:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:46,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:46,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:46,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375866989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375866989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,000 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:16:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,001 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:16:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375866989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375866997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375866998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,037 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/05f63486d53e4af98cbf94a777da4c7d 2024-12-05T05:16:47,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/464f93fac7f54d91badc2f42613488f9 is 50, key is test_row_0/B:col10/1733375804751/Put/seqid=0 2024-12-05T05:16:47,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741951_1127 (size=12001) 2024-12-05T05:16:47,125 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/464f93fac7f54d91badc2f42613488f9 2024-12-05T05:16:47,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375867121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375867121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/ead958cc1d0a45fe95132e1b8b84f9ff is 50, key is test_row_0/C:col10/1733375804751/Put/seqid=0 2024-12-05T05:16:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375867124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741952_1128 (size=12001) 2024-12-05T05:16:47,234 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/ead958cc1d0a45fe95132e1b8b84f9ff 2024-12-05T05:16:47,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/05f63486d53e4af98cbf94a777da4c7d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/05f63486d53e4af98cbf94a777da4c7d 2024-12-05T05:16:47,264 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/05f63486d53e4af98cbf94a777da4c7d, entries=150, sequenceid=51, filesize=30.2 K 2024-12-05T05:16:47,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/464f93fac7f54d91badc2f42613488f9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/464f93fac7f54d91badc2f42613488f9 2024-12-05T05:16:47,283 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/464f93fac7f54d91badc2f42613488f9, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T05:16:47,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/ead958cc1d0a45fe95132e1b8b84f9ff as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ead958cc1d0a45fe95132e1b8b84f9ff 2024-12-05T05:16:47,297 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ead958cc1d0a45fe95132e1b8b84f9ff, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T05:16:47,299 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 09d09c462e929eda7f34ea9ccdd56981 in 835ms, sequenceid=51, compaction requested=false 2024-12-05T05:16:47,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:47,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:47,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-05T05:16:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-12-05T05:16:47,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-05T05:16:47,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7690 sec 2024-12-05T05:16:47,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.7770 sec 2024-12-05T05:16:47,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T05:16:47,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:47,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:47,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:47,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:47,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:47,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:47,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:47,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052aff7cfe73a34018921832d2d6ff54a7_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375806986/Put/seqid=0 2024-12-05T05:16:47,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741953_1129 (size=12154) 2024-12-05T05:16:47,414 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:47,421 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052aff7cfe73a34018921832d2d6ff54a7_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052aff7cfe73a34018921832d2d6ff54a7_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:47,422 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/ac4c684f63074edd82e2a11e75ee0ff0, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:47,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/ac4c684f63074edd82e2a11e75ee0ff0 is 175, key is test_row_0/A:col10/1733375806986/Put/seqid=0 2024-12-05T05:16:47,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375867437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,450 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375867445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375867445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741954_1130 (size=30955) 2024-12-05T05:16:47,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375867554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,561 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375867565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375867565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375867775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375867775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375867776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:47,868 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/ac4c684f63074edd82e2a11e75ee0ff0 2024-12-05T05:16:47,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/5670a0aaa8c14b74a6ce29e7c75e0ba0 is 50, key is test_row_0/B:col10/1733375806986/Put/seqid=0 2024-12-05T05:16:47,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741955_1131 (size=12001) 2024-12-05T05:16:47,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/5670a0aaa8c14b74a6ce29e7c75e0ba0 2024-12-05T05:16:47,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/517a33c0a13f4c908696cbba3fe91d97 is 50, key is test_row_0/C:col10/1733375806986/Put/seqid=0 2024-12-05T05:16:47,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741956_1132 (size=12001) 2024-12-05T05:16:48,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375868093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375868093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375868096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/517a33c0a13f4c908696cbba3fe91d97 2024-12-05T05:16:48,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/ac4c684f63074edd82e2a11e75ee0ff0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ac4c684f63074edd82e2a11e75ee0ff0 2024-12-05T05:16:48,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ac4c684f63074edd82e2a11e75ee0ff0, entries=150, sequenceid=63, filesize=30.2 K 2024-12-05T05:16:48,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/5670a0aaa8c14b74a6ce29e7c75e0ba0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5670a0aaa8c14b74a6ce29e7c75e0ba0 2024-12-05T05:16:48,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5670a0aaa8c14b74a6ce29e7c75e0ba0, entries=150, sequenceid=63, filesize=11.7 K 2024-12-05T05:16:48,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/517a33c0a13f4c908696cbba3fe91d97 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/517a33c0a13f4c908696cbba3fe91d97 2024-12-05T05:16:48,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/517a33c0a13f4c908696cbba3fe91d97, entries=150, sequenceid=63, filesize=11.7 K 2024-12-05T05:16:48,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 09d09c462e929eda7f34ea9ccdd56981 in 1078ms, sequenceid=63, compaction requested=true 2024-12-05T05:16:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:48,422 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:48,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:48,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:48,423 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:48,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:48,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:48,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:48,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:48,424 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:48,424 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/A is initiating minor compaction (all files) 2024-12-05T05:16:48,424 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/A in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:48,424 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/39aa317f5e894268927e92c8ebcf791a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/05f63486d53e4af98cbf94a777da4c7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ac4c684f63074edd82e2a11e75ee0ff0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=107.5 K 2024-12-05T05:16:48,424 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:48,425 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/39aa317f5e894268927e92c8ebcf791a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/05f63486d53e4af98cbf94a777da4c7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ac4c684f63074edd82e2a11e75ee0ff0] 2024-12-05T05:16:48,425 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:48,425 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/B is initiating minor compaction (all files) 2024-12-05T05:16:48,425 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/B in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:48,425 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a45a95332aa246f6a5629527bf0e746a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/464f93fac7f54d91badc2f42613488f9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5670a0aaa8c14b74a6ce29e7c75e0ba0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=32.9 K 2024-12-05T05:16:48,426 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39aa317f5e894268927e92c8ebcf791a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733375804544 2024-12-05T05:16:48,426 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a45a95332aa246f6a5629527bf0e746a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733375804553 2024-12-05T05:16:48,428 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 464f93fac7f54d91badc2f42613488f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733375804751 2024-12-05T05:16:48,429 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05f63486d53e4af98cbf94a777da4c7d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733375804751 2024-12-05T05:16:48,429 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 5670a0aaa8c14b74a6ce29e7c75e0ba0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375806986 2024-12-05T05:16:48,429 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac4c684f63074edd82e2a11e75ee0ff0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375806986 2024-12-05T05:16:48,445 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:48,447 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#B#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:48,447 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/aabceade6874459d9d1722f3d9fd844b is 50, key is test_row_0/B:col10/1733375806986/Put/seqid=0 2024-12-05T05:16:48,457 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412052cb11e8275fd4d689fce0c3d9c54d4f5_09d09c462e929eda7f34ea9ccdd56981 store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:48,461 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412052cb11e8275fd4d689fce0c3d9c54d4f5_09d09c462e929eda7f34ea9ccdd56981, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:48,461 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052cb11e8275fd4d689fce0c3d9c54d4f5_09d09c462e929eda7f34ea9ccdd56981 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:48,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741957_1133 (size=12104) 2024-12-05T05:16:48,511 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/aabceade6874459d9d1722f3d9fd844b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/aabceade6874459d9d1722f3d9fd844b 2024-12-05T05:16:48,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741958_1134 (size=4469) 2024-12-05T05:16:48,520 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#A#compaction#115 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:48,521 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/b495cdada6344b7c8ebfe982cae48a2b is 175, key is test_row_0/A:col10/1733375806986/Put/seqid=0 2024-12-05T05:16:48,523 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/B of 09d09c462e929eda7f34ea9ccdd56981 into aabceade6874459d9d1722f3d9fd844b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:48,524 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:48,524 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/B, priority=13, startTime=1733375808423; duration=0sec 2024-12-05T05:16:48,524 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:48,524 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:B 2024-12-05T05:16:48,524 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:48,525 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:48,525 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/C is initiating minor compaction (all files) 2024-12-05T05:16:48,525 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/C in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:48,526 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/567090afe8514cd5b2486eac8b6e792d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ead958cc1d0a45fe95132e1b8b84f9ff, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/517a33c0a13f4c908696cbba3fe91d97] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=32.9 K 2024-12-05T05:16:48,527 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 567090afe8514cd5b2486eac8b6e792d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733375804553 2024-12-05T05:16:48,527 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ead958cc1d0a45fe95132e1b8b84f9ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733375804751 2024-12-05T05:16:48,528 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 517a33c0a13f4c908696cbba3fe91d97, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375806986 2024-12-05T05:16:48,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741959_1135 (size=31058) 2024-12-05T05:16:48,558 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#C#compaction#116 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:48,559 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/91196877312446feadad3d743e3775ee is 50, key is test_row_0/C:col10/1733375806986/Put/seqid=0 2024-12-05T05:16:48,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741960_1136 (size=12104) 2024-12-05T05:16:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:48,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T05:16:48,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375807395, max timestamp=1733375808603Num uniques 859; 2024-12-05T05:16:48,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375807395, max timestamp=1733375808603Num uniques 859; 2024-12-05T05:16:48,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:48,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:16:48,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:48,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:48,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:48,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:48,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:48,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:48,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ff570749aab54191baf6025a84820ba7_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375808604/Put/seqid=0 2024-12-05T05:16:48,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375868661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375868668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375868668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741961_1137 (size=12154) 2024-12-05T05:16:48,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:48,680 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ff570749aab54191baf6025a84820ba7_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff570749aab54191baf6025a84820ba7_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:48,683 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/fd95b2d88603489e99ba0a07b128d4da, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:48,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/fd95b2d88603489e99ba0a07b128d4da is 175, key is test_row_0/A:col10/1733375808604/Put/seqid=0 2024-12-05T05:16:48,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741962_1138 (size=30955) 2024-12-05T05:16:48,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/fd95b2d88603489e99ba0a07b128d4da 2024-12-05T05:16:48,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-05T05:16:48,704 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:48,705 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-12-05T05:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T05:16:48,708 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:48,709 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:48,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:48,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/2bae37dd525d42ebad26fc86a1d56ee8 is 50, key is test_row_0/B:col10/1733375808604/Put/seqid=0 2024-12-05T05:16:48,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741963_1139 (size=12001) 2024-12-05T05:16:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375868772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375868772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375868772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T05:16:48,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T05:16:48,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:48,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:48,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:48,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:48,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:48,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:48,963 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/b495cdada6344b7c8ebfe982cae48a2b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b495cdada6344b7c8ebfe982cae48a2b 2024-12-05T05:16:48,976 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/A of 09d09c462e929eda7f34ea9ccdd56981 into b495cdada6344b7c8ebfe982cae48a2b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:48,976 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:48,976 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/A, priority=13, startTime=1733375808422; duration=0sec 2024-12-05T05:16:48,977 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:48,977 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:A 2024-12-05T05:16:48,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375868983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375868983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:48,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375868984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:48,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:48,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:48,999 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/91196877312446feadad3d743e3775ee as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/91196877312446feadad3d743e3775ee 2024-12-05T05:16:49,014 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/C of 09d09c462e929eda7f34ea9ccdd56981 into 91196877312446feadad3d743e3775ee(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:49,015 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:49,015 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/C, priority=13, startTime=1733375808423; duration=0sec 2024-12-05T05:16:49,015 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:49,015 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:C 2024-12-05T05:16:49,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T05:16:49,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:49,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375869013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,019 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:16:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T05:16:49,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375869023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,025 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:16:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/2bae37dd525d42ebad26fc86a1d56ee8 2024-12-05T05:16:49,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/2a6a65ad0ee94e0b92c8830f3b3f4762 is 50, key is test_row_0/C:col10/1733375808604/Put/seqid=0 2024-12-05T05:16:49,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741964_1140 (size=12001) 2024-12-05T05:16:49,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/2a6a65ad0ee94e0b92c8830f3b3f4762 2024-12-05T05:16:49,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/fd95b2d88603489e99ba0a07b128d4da as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/fd95b2d88603489e99ba0a07b128d4da 2024-12-05T05:16:49,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T05:16:49,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:49,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/fd95b2d88603489e99ba0a07b128d4da, entries=150, sequenceid=86, filesize=30.2 K 2024-12-05T05:16:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/2bae37dd525d42ebad26fc86a1d56ee8 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2bae37dd525d42ebad26fc86a1d56ee8 2024-12-05T05:16:49,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2bae37dd525d42ebad26fc86a1d56ee8, entries=150, sequenceid=86, filesize=11.7 K 2024-12-05T05:16:49,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/2a6a65ad0ee94e0b92c8830f3b3f4762 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/2a6a65ad0ee94e0b92c8830f3b3f4762 2024-12-05T05:16:49,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/2a6a65ad0ee94e0b92c8830f3b3f4762, entries=150, sequenceid=86, filesize=11.7 K 2024-12-05T05:16:49,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=114.05 KB/116790 for 09d09c462e929eda7f34ea9ccdd56981 in 594ms, sequenceid=86, compaction requested=false 2024-12-05T05:16:49,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:49,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:49,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T05:16:49,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:49,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:49,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:49,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:49,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:49,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:49,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f10c861eeb174a33a85d600b5eb8a147_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375809293/Put/seqid=0 2024-12-05T05:16:49,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375869311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375869312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375869314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T05:16:49,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T05:16:49,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741965_1141 (size=14594) 2024-12-05T05:16:49,337 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:49,345 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f10c861eeb174a33a85d600b5eb8a147_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f10c861eeb174a33a85d600b5eb8a147_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:49,347 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/55aa6cc29ec84401a08e2d2762b4e648, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:49,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/55aa6cc29ec84401a08e2d2762b4e648 is 175, key is test_row_0/A:col10/1733375809293/Put/seqid=0 2024-12-05T05:16:49,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741966_1142 (size=39549) 2024-12-05T05:16:49,369 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=109, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/55aa6cc29ec84401a08e2d2762b4e648 2024-12-05T05:16:49,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/0425e36e1c5c4858bce70e538635c672 is 50, key is test_row_0/B:col10/1733375809293/Put/seqid=0 2024-12-05T05:16:49,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741967_1143 (size=12001) 2024-12-05T05:16:49,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/0425e36e1c5c4858bce70e538635c672 2024-12-05T05:16:49,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/dded4e4c8aef4a2a8429f0797251da06 is 50, key is test_row_0/C:col10/1733375809293/Put/seqid=0 2024-12-05T05:16:49,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375869423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375869424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375869424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741968_1144 (size=12001) 2024-12-05T05:16:49,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/dded4e4c8aef4a2a8429f0797251da06 2024-12-05T05:16:49,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/55aa6cc29ec84401a08e2d2762b4e648 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/55aa6cc29ec84401a08e2d2762b4e648 2024-12-05T05:16:49,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/55aa6cc29ec84401a08e2d2762b4e648, entries=200, sequenceid=109, filesize=38.6 K 2024-12-05T05:16:49,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/0425e36e1c5c4858bce70e538635c672 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0425e36e1c5c4858bce70e538635c672 2024-12-05T05:16:49,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T05:16:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:49,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:49,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0425e36e1c5c4858bce70e538635c672, entries=150, sequenceid=109, filesize=11.7 K 2024-12-05T05:16:49,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/dded4e4c8aef4a2a8429f0797251da06 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/dded4e4c8aef4a2a8429f0797251da06 2024-12-05T05:16:49,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/dded4e4c8aef4a2a8429f0797251da06, entries=150, sequenceid=109, filesize=11.7 K 2024-12-05T05:16:49,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 09d09c462e929eda7f34ea9ccdd56981 in 200ms, sequenceid=109, compaction requested=true 2024-12-05T05:16:49,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:49,497 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:49,497 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:49,497 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:49,498 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:49,498 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/B is initiating minor compaction (all files) 2024-12-05T05:16:49,498 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/B in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,499 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/aabceade6874459d9d1722f3d9fd844b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2bae37dd525d42ebad26fc86a1d56ee8, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0425e36e1c5c4858bce70e538635c672] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=35.3 K 2024-12-05T05:16:49,499 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:49,499 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/A is initiating minor compaction (all files) 2024-12-05T05:16:49,499 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/A in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,499 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b495cdada6344b7c8ebfe982cae48a2b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/fd95b2d88603489e99ba0a07b128d4da, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/55aa6cc29ec84401a08e2d2762b4e648] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=99.2 K 2024-12-05T05:16:49,499 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,499 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b495cdada6344b7c8ebfe982cae48a2b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/fd95b2d88603489e99ba0a07b128d4da, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/55aa6cc29ec84401a08e2d2762b4e648] 2024-12-05T05:16:49,500 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting aabceade6874459d9d1722f3d9fd844b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375806986 2024-12-05T05:16:49,500 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b495cdada6344b7c8ebfe982cae48a2b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375806986 2024-12-05T05:16:49,501 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd95b2d88603489e99ba0a07b128d4da, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733375807395 2024-12-05T05:16:49,501 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bae37dd525d42ebad26fc86a1d56ee8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733375807395 2024-12-05T05:16:49,502 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55aa6cc29ec84401a08e2d2762b4e648, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375808659 2024-12-05T05:16:49,502 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0425e36e1c5c4858bce70e538635c672, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375808659 2024-12-05T05:16:49,533 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#B#compaction#123 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:49,536 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/a244c9f390c94bb8b40ca1cee27f6279 is 50, key is test_row_0/B:col10/1733375809293/Put/seqid=0 2024-12-05T05:16:49,544 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:49,577 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205b16b0d3dd3514db9ac25559e86904aed_09d09c462e929eda7f34ea9ccdd56981 store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:49,580 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205b16b0d3dd3514db9ac25559e86904aed_09d09c462e929eda7f34ea9ccdd56981, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:49,580 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b16b0d3dd3514db9ac25559e86904aed_09d09c462e929eda7f34ea9ccdd56981 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:49,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741969_1145 (size=12207) 2024-12-05T05:16:49,597 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/a244c9f390c94bb8b40ca1cee27f6279 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a244c9f390c94bb8b40ca1cee27f6279 2024-12-05T05:16:49,625 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/B of 09d09c462e929eda7f34ea9ccdd56981 into a244c9f390c94bb8b40ca1cee27f6279(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:49,625 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:49,625 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/B, priority=13, startTime=1733375809497; duration=0sec 2024-12-05T05:16:49,625 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:49,625 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:B 2024-12-05T05:16:49,625 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:49,628 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:49,628 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/C is initiating minor compaction (all files) 2024-12-05T05:16:49,628 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/C in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,628 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/91196877312446feadad3d743e3775ee, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/2a6a65ad0ee94e0b92c8830f3b3f4762, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/dded4e4c8aef4a2a8429f0797251da06] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=35.3 K 2024-12-05T05:16:49,629 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 91196877312446feadad3d743e3775ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375806986 2024-12-05T05:16:49,629 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a6a65ad0ee94e0b92c8830f3b3f4762, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1733375807395 2024-12-05T05:16:49,630 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting dded4e4c8aef4a2a8429f0797251da06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375808659 2024-12-05T05:16:49,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-12-05T05:16:49,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,635 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:16:49,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:49,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:49,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:49,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:49,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741970_1146 (size=4469) 2024-12-05T05:16:49,650 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#A#compaction#124 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:49,650 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/bb4e9704f20546ed8982d49070b48a54 is 175, key is test_row_0/A:col10/1733375809293/Put/seqid=0 2024-12-05T05:16:49,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120585fba102a4db4a858b02e9fc54b914a0_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375809309/Put/seqid=0 2024-12-05T05:16:49,654 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#C#compaction#125 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:49,654 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/e538550b95864eaab453e7e346f75273 is 50, key is test_row_0/C:col10/1733375809293/Put/seqid=0 2024-12-05T05:16:49,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741971_1147 (size=31161) 2024-12-05T05:16:49,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741972_1148 (size=12207) 2024-12-05T05:16:49,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375809674, max timestamp=1733375809692Num uniques 859; 2024-12-05T05:16:49,695 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:49,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375809674, max timestamp=1733375809692Num uniques 859; 2024-12-05T05:16:49,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375809674, max timestamp=1733375809692Num uniques 859; 2024-12-05T05:16:49,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:49,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:49,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741973_1149 (size=12154) 2024-12-05T05:16:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:49,718 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120585fba102a4db4a858b02e9fc54b914a0_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120585fba102a4db4a858b02e9fc54b914a0_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:49,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/b70285c5bd9d43b9bf372d17a8fe6c2a, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/b70285c5bd9d43b9bf372d17a8fe6c2a is 175, key is test_row_0/A:col10/1733375809309/Put/seqid=0 2024-12-05T05:16:49,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741974_1150 (size=30955) 2024-12-05T05:16:49,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375869742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375869745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,750 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=121, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/b70285c5bd9d43b9bf372d17a8fe6c2a 2024-12-05T05:16:49,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375869747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/f688e5ed55994e06970e4cbc8e55f565 is 50, key is test_row_0/B:col10/1733375809309/Put/seqid=0 2024-12-05T05:16:49,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741975_1151 (size=12001) 2024-12-05T05:16:49,807 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/f688e5ed55994e06970e4cbc8e55f565 2024-12-05T05:16:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T05:16:49,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/68cfca1edd4b4fd1a69a57f6c8b410af is 50, key is test_row_0/C:col10/1733375809309/Put/seqid=0 2024-12-05T05:16:49,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375869854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375869860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741976_1152 (size=12001) 2024-12-05T05:16:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,863 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/68cfca1edd4b4fd1a69a57f6c8b410af 2024-12-05T05:16:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375869863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:49,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/b70285c5bd9d43b9bf372d17a8fe6c2a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b70285c5bd9d43b9bf372d17a8fe6c2a 2024-12-05T05:16:49,890 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b70285c5bd9d43b9bf372d17a8fe6c2a, entries=150, sequenceid=121, filesize=30.2 K 2024-12-05T05:16:49,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/f688e5ed55994e06970e4cbc8e55f565 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f688e5ed55994e06970e4cbc8e55f565 2024-12-05T05:16:49,899 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f688e5ed55994e06970e4cbc8e55f565, entries=150, sequenceid=121, filesize=11.7 K 2024-12-05T05:16:49,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/68cfca1edd4b4fd1a69a57f6c8b410af as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/68cfca1edd4b4fd1a69a57f6c8b410af 2024-12-05T05:16:49,907 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/68cfca1edd4b4fd1a69a57f6c8b410af, entries=150, sequenceid=121, filesize=11.7 K 2024-12-05T05:16:49,909 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 09d09c462e929eda7f34ea9ccdd56981 in 274ms, sequenceid=121, compaction requested=false 2024-12-05T05:16:49,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:49,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:49,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-12-05T05:16:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-12-05T05:16:49,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-12-05T05:16:49,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2020 sec 2024-12-05T05:16:49,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 1.2080 sec 2024-12-05T05:16:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:50,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-05T05:16:50,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:50,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:50,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:50,067 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:50,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:50,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:50,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205040c563976ac4c388ca6934554d18bd6_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375809740/Put/seqid=0 2024-12-05T05:16:50,087 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/bb4e9704f20546ed8982d49070b48a54 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/bb4e9704f20546ed8982d49070b48a54 2024-12-05T05:16:50,094 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/e538550b95864eaab453e7e346f75273 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e538550b95864eaab453e7e346f75273 2024-12-05T05:16:50,105 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/A of 09d09c462e929eda7f34ea9ccdd56981 into bb4e9704f20546ed8982d49070b48a54(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:50,105 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:50,105 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/A, priority=13, startTime=1733375809497; duration=0sec 2024-12-05T05:16:50,106 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:50,106 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:A 2024-12-05T05:16:50,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375870092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741977_1153 (size=14794) 2024-12-05T05:16:50,112 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/C of 09d09c462e929eda7f34ea9ccdd56981 into e538550b95864eaab453e7e346f75273(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:50,113 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:50,113 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/C, priority=13, startTime=1733375809497; duration=0sec 2024-12-05T05:16:50,113 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:50,113 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:C 2024-12-05T05:16:50,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375870114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375870115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375870212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375870233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375870234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375870423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375870443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375870444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,516 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:50,527 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205040c563976ac4c388ca6934554d18bd6_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205040c563976ac4c388ca6934554d18bd6_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:50,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/96fcb07dfb704e4f98e2828343b36590, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:50,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/96fcb07dfb704e4f98e2828343b36590 is 175, key is test_row_0/A:col10/1733375809740/Put/seqid=0 2024-12-05T05:16:50,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741978_1154 (size=39749) 2024-12-05T05:16:50,557 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/96fcb07dfb704e4f98e2828343b36590 2024-12-05T05:16:50,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/bec73c16b43d40c4a7fdc8b643d023c4 is 50, key is test_row_0/B:col10/1733375809740/Put/seqid=0 2024-12-05T05:16:50,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741979_1155 (size=12151) 2024-12-05T05:16:50,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/bec73c16b43d40c4a7fdc8b643d023c4 2024-12-05T05:16:50,684 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/e053d8d741534db991a1ee0de6ac59a0 is 50, key is test_row_0/C:col10/1733375809740/Put/seqid=0 2024-12-05T05:16:50,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741980_1156 (size=12151) 2024-12-05T05:16:50,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/e053d8d741534db991a1ee0de6ac59a0 2024-12-05T05:16:50,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/96fcb07dfb704e4f98e2828343b36590 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/96fcb07dfb704e4f98e2828343b36590 2024-12-05T05:16:50,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/96fcb07dfb704e4f98e2828343b36590, entries=200, sequenceid=157, filesize=38.8 K 2024-12-05T05:16:50,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375870733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/bec73c16b43d40c4a7fdc8b643d023c4 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/bec73c16b43d40c4a7fdc8b643d023c4 2024-12-05T05:16:50,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/bec73c16b43d40c4a7fdc8b643d023c4, entries=150, sequenceid=157, filesize=11.9 K 2024-12-05T05:16:50,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/e053d8d741534db991a1ee0de6ac59a0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e053d8d741534db991a1ee0de6ac59a0 2024-12-05T05:16:50,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375870753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:50,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375870763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:50,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:50,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e053d8d741534db991a1ee0de6ac59a0, entries=150, sequenceid=157, filesize=11.9 K 2024-12-05T05:16:50,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 09d09c462e929eda7f34ea9ccdd56981 in 716ms, sequenceid=157, compaction requested=true 2024-12-05T05:16:50,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:50,783 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:50,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:50,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:50,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:50,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:50,784 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:50,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:50,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:50,785 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:50,785 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/A is initiating minor compaction (all files) 2024-12-05T05:16:50,785 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/A in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:50,785 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/bb4e9704f20546ed8982d49070b48a54, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b70285c5bd9d43b9bf372d17a8fe6c2a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/96fcb07dfb704e4f98e2828343b36590] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=99.5 K 2024-12-05T05:16:50,786 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:50,786 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/bb4e9704f20546ed8982d49070b48a54, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b70285c5bd9d43b9bf372d17a8fe6c2a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/96fcb07dfb704e4f98e2828343b36590] 2024-12-05T05:16:50,786 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:50,786 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/B is initiating minor compaction (all files) 2024-12-05T05:16:50,786 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/B in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:50,786 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a244c9f390c94bb8b40ca1cee27f6279, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f688e5ed55994e06970e4cbc8e55f565, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/bec73c16b43d40c4a7fdc8b643d023c4] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=35.5 K 2024-12-05T05:16:50,787 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb4e9704f20546ed8982d49070b48a54, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375808659 2024-12-05T05:16:50,787 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a244c9f390c94bb8b40ca1cee27f6279, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375808659 2024-12-05T05:16:50,787 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b70285c5bd9d43b9bf372d17a8fe6c2a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733375809296 2024-12-05T05:16:50,788 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f688e5ed55994e06970e4cbc8e55f565, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733375809296 2024-12-05T05:16:50,788 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting bec73c16b43d40c4a7fdc8b643d023c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733375809734 2024-12-05T05:16:50,790 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96fcb07dfb704e4f98e2828343b36590, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733375809734 2024-12-05T05:16:50,821 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:50,829 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#B#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:50,830 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/9c0c791e352c4e90a3ee1ecad5fa604f is 50, key is test_row_0/B:col10/1733375809740/Put/seqid=0 2024-12-05T05:16:50,833 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412055a5ee0098a2949bdb6c2f2ecdc393dce_09d09c462e929eda7f34ea9ccdd56981 store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:50,835 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412055a5ee0098a2949bdb6c2f2ecdc393dce_09d09c462e929eda7f34ea9ccdd56981, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:50,835 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055a5ee0098a2949bdb6c2f2ecdc393dce_09d09c462e929eda7f34ea9ccdd56981 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-12-05T05:16:50,853 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:50,854 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:50,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-12-05T05:16:50,857 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:50,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T05:16:50,858 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:50,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:50,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741981_1157 (size=12459) 2024-12-05T05:16:50,887 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/9c0c791e352c4e90a3ee1ecad5fa604f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9c0c791e352c4e90a3ee1ecad5fa604f 2024-12-05T05:16:50,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741982_1158 (size=4469) 2024-12-05T05:16:50,888 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#A#compaction#132 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:50,889 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/668ddaaedb0f48d19a1a6627a64fa986 is 175, key is test_row_0/A:col10/1733375809740/Put/seqid=0 2024-12-05T05:16:50,900 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/B of 09d09c462e929eda7f34ea9ccdd56981 into 9c0c791e352c4e90a3ee1ecad5fa604f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:50,900 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:50,900 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/B, priority=13, startTime=1733375810784; duration=0sec 2024-12-05T05:16:50,900 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:50,900 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:B 2024-12-05T05:16:50,900 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:50,906 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:50,906 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/C is initiating minor compaction (all files) 2024-12-05T05:16:50,906 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/C in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:50,906 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e538550b95864eaab453e7e346f75273, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/68cfca1edd4b4fd1a69a57f6c8b410af, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e053d8d741534db991a1ee0de6ac59a0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=35.5 K 2024-12-05T05:16:50,907 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e538550b95864eaab453e7e346f75273, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375808659 2024-12-05T05:16:50,908 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 68cfca1edd4b4fd1a69a57f6c8b410af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1733375809296 2024-12-05T05:16:50,909 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e053d8d741534db991a1ee0de6ac59a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733375809734 2024-12-05T05:16:50,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741983_1159 (size=31413) 2024-12-05T05:16:50,921 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#C#compaction#134 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:50,921 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/23bb963814c349448bf237467e9ff5e2 is 50, key is test_row_0/C:col10/1733375809740/Put/seqid=0 2024-12-05T05:16:50,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:16:50,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T05:16:50,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741984_1160 (size=12459) 2024-12-05T05:16:50,952 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/23bb963814c349448bf237467e9ff5e2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/23bb963814c349448bf237467e9ff5e2 2024-12-05T05:16:50,961 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/C of 09d09c462e929eda7f34ea9ccdd56981 into 23bb963814c349448bf237467e9ff5e2(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:50,961 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:50,961 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/C, priority=13, startTime=1733375810784; duration=0sec 2024-12-05T05:16:50,961 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:50,961 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:C 2024-12-05T05:16:50,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T05:16:51,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-12-05T05:16:51,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:51,012 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:16:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:51,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:51,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120535799a99ab9f4cbeb7cc5d1a37b0d25e_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375810103/Put/seqid=0 2024-12-05T05:16:51,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741985_1161 (size=12304) 2024-12-05T05:16:51,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:51,074 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120535799a99ab9f4cbeb7cc5d1a37b0d25e_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120535799a99ab9f4cbeb7cc5d1a37b0d25e_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/6aa88a569187444bae0423cb6c3fdb3d, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:51,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/6aa88a569187444bae0423cb6c3fdb3d is 175, key is test_row_0/A:col10/1733375810103/Put/seqid=0 2024-12-05T05:16:51,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741986_1162 (size=31105) 2024-12-05T05:16:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T05:16:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:51,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:51,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375811277, max timestamp=1733375811298Num uniques 859; 2024-12-05T05:16:51,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:51,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375811277, max timestamp=1733375811298Num uniques 859; 2024-12-05T05:16:51,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:51,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375811277, max timestamp=1733375811298Num uniques 859; 2024-12-05T05:16:51,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:51,321 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/668ddaaedb0f48d19a1a6627a64fa986 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/668ddaaedb0f48d19a1a6627a64fa986 2024-12-05T05:16:51,330 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/A of 09d09c462e929eda7f34ea9ccdd56981 into 668ddaaedb0f48d19a1a6627a64fa986(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:51,330 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:51,330 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/A, priority=13, startTime=1733375810783; duration=0sec 2024-12-05T05:16:51,331 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:51,331 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:A 2024-12-05T05:16:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375871342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375871343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375871343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375871452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375871453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375871453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,481 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/6aa88a569187444bae0423cb6c3fdb3d 2024-12-05T05:16:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T05:16:51,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/f6dfadfe3f9a43b9bfd717605bdc1269 is 50, key is test_row_0/B:col10/1733375810103/Put/seqid=0 2024-12-05T05:16:51,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741987_1163 (size=12151) 2024-12-05T05:16:51,536 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/f6dfadfe3f9a43b9bfd717605bdc1269 2024-12-05T05:16:51,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/7e79af0a83ed43cba0d8aec59e6cafdd is 50, key is test_row_0/C:col10/1733375810103/Put/seqid=0 2024-12-05T05:16:51,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741988_1164 (size=12151) 2024-12-05T05:16:51,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375871663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,670 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375871663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375871664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375871973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,987 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/7e79af0a83ed43cba0d8aec59e6cafdd 2024-12-05T05:16:51,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375871983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375871983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:51,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T05:16:51,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/6aa88a569187444bae0423cb6c3fdb3d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6aa88a569187444bae0423cb6c3fdb3d 2024-12-05T05:16:52,001 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6aa88a569187444bae0423cb6c3fdb3d, entries=150, sequenceid=171, filesize=30.4 K 2024-12-05T05:16:52,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/f6dfadfe3f9a43b9bfd717605bdc1269 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f6dfadfe3f9a43b9bfd717605bdc1269 2024-12-05T05:16:52,010 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f6dfadfe3f9a43b9bfd717605bdc1269, entries=150, sequenceid=171, filesize=11.9 K 2024-12-05T05:16:52,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/7e79af0a83ed43cba0d8aec59e6cafdd as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/7e79af0a83ed43cba0d8aec59e6cafdd 2024-12-05T05:16:52,032 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/7e79af0a83ed43cba0d8aec59e6cafdd, entries=150, sequenceid=171, filesize=11.9 K 2024-12-05T05:16:52,034 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 09d09c462e929eda7f34ea9ccdd56981 in 1022ms, sequenceid=171, compaction requested=false 2024-12-05T05:16:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:52,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-12-05T05:16:52,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-12-05T05:16:52,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-05T05:16:52,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1780 sec 2024-12-05T05:16:52,041 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.1850 sec 2024-12-05T05:16:52,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:52,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-05T05:16:52,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:52,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:52,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:52,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:52,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:52,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:52,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205530286856fb4449db1f324a2d2318d40_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375811341/Put/seqid=0 2024-12-05T05:16:52,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741989_1165 (size=14794) 2024-12-05T05:16:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375872509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375872511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375872520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375872624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375872626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375872634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,636 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375872834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375872834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:52,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375872843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:52,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:52,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:52,915 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205530286856fb4449db1f324a2d2318d40_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205530286856fb4449db1f324a2d2318d40_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:52,921 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/1b3eb626d5bb4d7684a813d35b9182ec, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:52,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/1b3eb626d5bb4d7684a813d35b9182ec is 175, key is test_row_0/A:col10/1733375811341/Put/seqid=0 2024-12-05T05:16:52,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741990_1166 (size=39749) 2024-12-05T05:16:52,954 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/1b3eb626d5bb4d7684a813d35b9182ec 2024-12-05T05:16:52,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/7224fc50cbb643c58138b7d702baa5e3 is 50, key is test_row_0/B:col10/1733375811341/Put/seqid=0 2024-12-05T05:16:53,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-05T05:16:53,003 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:53,004 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-12-05T05:16:53,006 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T05:16:53,008 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:53,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:53,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741991_1167 (size=12151) 2024-12-05T05:16:53,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375873035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,036 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:16:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375873053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,056 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8298 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:16:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T05:16:53,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375873143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375873153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T05:16:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375873163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T05:16:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T05:16:53,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/7224fc50cbb643c58138b7d702baa5e3 2024-12-05T05:16:53,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/1b79cdcef2c04738a05f9cc6cb3d034a is 50, key is test_row_0/C:col10/1733375811341/Put/seqid=0 2024-12-05T05:16:53,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741992_1168 (size=12151) 2024-12-05T05:16:53,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/1b79cdcef2c04738a05f9cc6cb3d034a 2024-12-05T05:16:53,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T05:16:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:53,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/1b3eb626d5bb4d7684a813d35b9182ec as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/1b3eb626d5bb4d7684a813d35b9182ec 2024-12-05T05:16:53,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/1b3eb626d5bb4d7684a813d35b9182ec, entries=200, sequenceid=209, filesize=38.8 K 2024-12-05T05:16:53,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/7224fc50cbb643c58138b7d702baa5e3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/7224fc50cbb643c58138b7d702baa5e3 2024-12-05T05:16:53,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/7224fc50cbb643c58138b7d702baa5e3, entries=150, sequenceid=209, filesize=11.9 K 2024-12-05T05:16:53,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/1b79cdcef2c04738a05f9cc6cb3d034a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1b79cdcef2c04738a05f9cc6cb3d034a 2024-12-05T05:16:53,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1b79cdcef2c04738a05f9cc6cb3d034a, entries=150, sequenceid=209, filesize=11.9 K 2024-12-05T05:16:53,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 09d09c462e929eda7f34ea9ccdd56981 in 1033ms, sequenceid=209, compaction requested=true 2024-12-05T05:16:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:53,520 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:53,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:53,520 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:53,521 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:53,522 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/B is initiating minor compaction (all files) 2024-12-05T05:16:53,522 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/B in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,522 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9c0c791e352c4e90a3ee1ecad5fa604f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f6dfadfe3f9a43b9bfd717605bdc1269, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/7224fc50cbb643c58138b7d702baa5e3] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=35.9 K 2024-12-05T05:16:53,522 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:53,522 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/A is initiating minor compaction (all files) 2024-12-05T05:16:53,522 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/A in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,522 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/668ddaaedb0f48d19a1a6627a64fa986, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6aa88a569187444bae0423cb6c3fdb3d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/1b3eb626d5bb4d7684a813d35b9182ec] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=99.9 K 2024-12-05T05:16:53,522 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,522 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/668ddaaedb0f48d19a1a6627a64fa986, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6aa88a569187444bae0423cb6c3fdb3d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/1b3eb626d5bb4d7684a813d35b9182ec] 2024-12-05T05:16:53,523 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c0c791e352c4e90a3ee1ecad5fa604f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733375809734 2024-12-05T05:16:53,523 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 668ddaaedb0f48d19a1a6627a64fa986, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733375809734 2024-12-05T05:16:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:53,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:53,523 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aa88a569187444bae0423cb6c3fdb3d, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375810069 2024-12-05T05:16:53,523 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f6dfadfe3f9a43b9bfd717605bdc1269, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375810069 2024-12-05T05:16:53,524 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b3eb626d5bb4d7684a813d35b9182ec, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733375811321 2024-12-05T05:16:53,524 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7224fc50cbb643c58138b7d702baa5e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733375811340 2024-12-05T05:16:53,537 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:53,538 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/18b9cc42b0484f87b01c3bd9119e0ce3 is 50, key is test_row_0/B:col10/1733375811341/Put/seqid=0 2024-12-05T05:16:53,545 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:53,568 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412055f76a374a6e44da1a0632b018ea0623b_09d09c462e929eda7f34ea9ccdd56981 store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:53,570 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412055f76a374a6e44da1a0632b018ea0623b_09d09c462e929eda7f34ea9ccdd56981, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:53,570 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055f76a374a6e44da1a0632b018ea0623b_09d09c462e929eda7f34ea9ccdd56981 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:53,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741993_1169 (size=12561) 2024-12-05T05:16:53,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741994_1170 (size=4469) 2024-12-05T05:16:53,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-12-05T05:16:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:53,627 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-05T05:16:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:53,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:53,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:53,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:53,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T05:16:53,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120592da0344cf4846f081a0c4b01c216b84_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375812511/Put/seqid=0 2024-12-05T05:16:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:53,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:53,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741995_1171 (size=12304) 2024-12-05T05:16:53,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:53,694 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120592da0344cf4846f081a0c4b01c216b84_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120592da0344cf4846f081a0c4b01c216b84_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:53,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/954f81bc0bab4067a8ddec7d322a2094, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/954f81bc0bab4067a8ddec7d322a2094 is 175, key is test_row_0/A:col10/1733375812511/Put/seqid=0 2024-12-05T05:16:53,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741996_1172 (size=31105) 2024-12-05T05:16:53,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733375813703, max timestamp=1733375813720Num uniques 859; 2024-12-05T05:16:53,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T05:16:53,725 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=218, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/954f81bc0bab4067a8ddec7d322a2094 2024-12-05T05:16:53,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733375813703, max timestamp=1733375813720Num uniques 859; 2024-12-05T05:16:53,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T05:16:53,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733375813703, max timestamp=1733375813720Num uniques 859; 2024-12-05T05:16:53,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T05:16:53,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/b3274e508c444893ac107833c9b34453 is 50, key is test_row_0/B:col10/1733375812511/Put/seqid=0 2024-12-05T05:16:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375873785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375873786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375873789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,805 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741997_1173 (size=12151) 2024-12-05T05:16:53,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375873894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375873894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:53,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375873914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:53,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:53,986 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/18b9cc42b0484f87b01c3bd9119e0ce3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/18b9cc42b0484f87b01c3bd9119e0ce3 2024-12-05T05:16:54,005 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/B of 09d09c462e929eda7f34ea9ccdd56981 into 18b9cc42b0484f87b01c3bd9119e0ce3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:54,005 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:54,005 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/B, priority=13, startTime=1733375813520; duration=0sec 2024-12-05T05:16:54,005 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:54,005 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:B 2024-12-05T05:16:54,005 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:54,017 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#A#compaction#142 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:54,018 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/cff379d6cae9432986ac7f5ee68f5920 is 175, key is test_row_0/A:col10/1733375811341/Put/seqid=0 2024-12-05T05:16:54,019 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:54,019 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/C is initiating minor compaction (all files) 2024-12-05T05:16:54,019 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/C in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:54,020 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/23bb963814c349448bf237467e9ff5e2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/7e79af0a83ed43cba0d8aec59e6cafdd, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1b79cdcef2c04738a05f9cc6cb3d034a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=35.9 K 2024-12-05T05:16:54,020 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 23bb963814c349448bf237467e9ff5e2, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1733375809734 2024-12-05T05:16:54,021 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e79af0a83ed43cba0d8aec59e6cafdd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375810069 2024-12-05T05:16:54,022 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b79cdcef2c04738a05f9cc6cb3d034a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733375811340 2024-12-05T05:16:54,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741998_1174 (size=31515) 2024-12-05T05:16:54,059 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/cff379d6cae9432986ac7f5ee68f5920 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/cff379d6cae9432986ac7f5ee68f5920 2024-12-05T05:16:54,060 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#C#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:54,061 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/df44226e9f314a5e9ca004e48940445c is 50, key is test_row_0/C:col10/1733375811341/Put/seqid=0 2024-12-05T05:16:54,073 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/A of 09d09c462e929eda7f34ea9ccdd56981 into cff379d6cae9432986ac7f5ee68f5920(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:54,073 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:54,073 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/A, priority=13, startTime=1733375813520; duration=0sec 2024-12-05T05:16:54,073 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:54,073 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:A 2024-12-05T05:16:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741999_1175 (size=12561) 2024-12-05T05:16:54,106 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/df44226e9f314a5e9ca004e48940445c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/df44226e9f314a5e9ca004e48940445c 2024-12-05T05:16:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375874103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375874103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,116 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/C of 09d09c462e929eda7f34ea9ccdd56981 into df44226e9f314a5e9ca004e48940445c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:54,116 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:54,117 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/C, priority=13, startTime=1733375813523; duration=0sec 2024-12-05T05:16:54,117 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:54,117 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:C 2024-12-05T05:16:54,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375874123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T05:16:54,206 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/b3274e508c444893ac107833c9b34453 2024-12-05T05:16:54,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/bbe3b02063284191a5097d9cfad34d2d is 50, key is test_row_0/C:col10/1733375812511/Put/seqid=0 2024-12-05T05:16:54,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742000_1176 (size=12151) 2024-12-05T05:16:54,254 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/bbe3b02063284191a5097d9cfad34d2d 2024-12-05T05:16:54,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/954f81bc0bab4067a8ddec7d322a2094 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/954f81bc0bab4067a8ddec7d322a2094 2024-12-05T05:16:54,267 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/954f81bc0bab4067a8ddec7d322a2094, entries=150, sequenceid=218, filesize=30.4 K 2024-12-05T05:16:54,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/b3274e508c444893ac107833c9b34453 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/b3274e508c444893ac107833c9b34453 2024-12-05T05:16:54,283 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/b3274e508c444893ac107833c9b34453, entries=150, sequenceid=218, filesize=11.9 K 2024-12-05T05:16:54,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/bbe3b02063284191a5097d9cfad34d2d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/bbe3b02063284191a5097d9cfad34d2d 2024-12-05T05:16:54,296 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/bbe3b02063284191a5097d9cfad34d2d, entries=150, sequenceid=218, filesize=11.9 K 2024-12-05T05:16:54,298 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 09d09c462e929eda7f34ea9ccdd56981 in 671ms, sequenceid=218, compaction requested=false 2024-12-05T05:16:54,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:54,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:54,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-12-05T05:16:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-12-05T05:16:54,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-12-05T05:16:54,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2930 sec 2024-12-05T05:16:54,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3020 sec 2024-12-05T05:16:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:54,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-05T05:16:54,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733375813772, max timestamp=1733375814419Num uniques 1118; 2024-12-05T05:16:54,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733375813772, max timestamp=1733375814419Num uniques 1118; 2024-12-05T05:16:54,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-05T05:16:54,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-05T05:16:54,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:54,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:54,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:54,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:54,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:54,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T05:16:54,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120507e5d3471d424677ad5952fa1b024b6c_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375814419/Put/seqid=0 2024-12-05T05:16:54,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375874465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375874466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742001_1177 (size=14894) 2024-12-05T05:16:54,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375874468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375874573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375874573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375874582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375874784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375874784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375874793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:54,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:54,885 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120507e5d3471d424677ad5952fa1b024b6c_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120507e5d3471d424677ad5952fa1b024b6c_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:54,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/c1513811bad146e084ffff81dca3b4a0, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:54,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/c1513811bad146e084ffff81dca3b4a0 is 175, key is test_row_0/A:col10/1733375814419/Put/seqid=0 2024-12-05T05:16:54,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742002_1178 (size=39849) 2024-12-05T05:16:54,907 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/c1513811bad146e084ffff81dca3b4a0 2024-12-05T05:16:54,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/6baa642623804686bbe7c4b1e5b6c96f is 50, key is test_row_0/B:col10/1733375814419/Put/seqid=0 2024-12-05T05:16:54,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742003_1179 (size=12251) 2024-12-05T05:16:55,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375875093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375875094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,100 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375875103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,107 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-12-05T05:16:55,154 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:55,154 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:55,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-05T05:16:55,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T05:16:55,157 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:55,158 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:55,158 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:55,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T05:16:55,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T05:16:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:55,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:55,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:55,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:55,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:16:55,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/6baa642623804686bbe7c4b1e5b6c96f 2024-12-05T05:16:55,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/61b97f69722249819d6ab63327c942bb is 50, key is test_row_0/C:col10/1733375814419/Put/seqid=0 2024-12-05T05:16:55,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742004_1180 (size=12251) 2024-12-05T05:16:55,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/61b97f69722249819d6ab63327c942bb 2024-12-05T05:16:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/c1513811bad146e084ffff81dca3b4a0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1513811bad146e084ffff81dca3b4a0 2024-12-05T05:16:55,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1513811bad146e084ffff81dca3b4a0, entries=200, sequenceid=259, filesize=38.9 K 2024-12-05T05:16:55,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/6baa642623804686bbe7c4b1e5b6c96f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/6baa642623804686bbe7c4b1e5b6c96f 2024-12-05T05:16:55,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/6baa642623804686bbe7c4b1e5b6c96f, entries=150, sequenceid=259, filesize=12.0 K 2024-12-05T05:16:55,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/61b97f69722249819d6ab63327c942bb as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/61b97f69722249819d6ab63327c942bb 2024-12-05T05:16:55,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/61b97f69722249819d6ab63327c942bb, entries=150, sequenceid=259, filesize=12.0 K 2024-12-05T05:16:55,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~199.06 KB/203840, currentSize=107.34 KB/109920 for 09d09c462e929eda7f34ea9ccdd56981 in 997ms, sequenceid=259, compaction requested=true 2024-12-05T05:16:55,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:55,419 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:55,419 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:55,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:55,420 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:55,420 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:55,420 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/A is initiating minor compaction (all files) 2024-12-05T05:16:55,420 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/B is initiating minor compaction (all files) 2024-12-05T05:16:55,420 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/A in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:55,420 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/B in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:55,420 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/cff379d6cae9432986ac7f5ee68f5920, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/954f81bc0bab4067a8ddec7d322a2094, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1513811bad146e084ffff81dca3b4a0] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=100.1 K 2024-12-05T05:16:55,420 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/18b9cc42b0484f87b01c3bd9119e0ce3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/b3274e508c444893ac107833c9b34453, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/6baa642623804686bbe7c4b1e5b6c96f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=36.1 K 2024-12-05T05:16:55,420 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:55,420 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/cff379d6cae9432986ac7f5ee68f5920, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/954f81bc0bab4067a8ddec7d322a2094, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1513811bad146e084ffff81dca3b4a0] 2024-12-05T05:16:55,424 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b9cc42b0484f87b01c3bd9119e0ce3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733375811340 2024-12-05T05:16:55,424 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting cff379d6cae9432986ac7f5ee68f5920, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733375811340 2024-12-05T05:16:55,425 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 954f81bc0bab4067a8ddec7d322a2094, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733375812499 2024-12-05T05:16:55,425 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b3274e508c444893ac107833c9b34453, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733375812499 2024-12-05T05:16:55,425 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1513811bad146e084ffff81dca3b4a0, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375813772 2024-12-05T05:16:55,426 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6baa642623804686bbe7c4b1e5b6c96f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375813772 2024-12-05T05:16:55,437 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:55,445 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120594c0b9c070144cb28ad7a86754f1ce44_09d09c462e929eda7f34ea9ccdd56981 store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:55,446 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#B#compaction#151 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:55,447 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/cbf1476667f749f2b0f05f01d53cb101 is 50, key is test_row_0/B:col10/1733375814419/Put/seqid=0 2024-12-05T05:16:55,447 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120594c0b9c070144cb28ad7a86754f1ce44_09d09c462e929eda7f34ea9ccdd56981, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:55,447 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120594c0b9c070144cb28ad7a86754f1ce44_09d09c462e929eda7f34ea9ccdd56981 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:55,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742005_1181 (size=12763) 2024-12-05T05:16:55,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742006_1182 (size=4469) 2024-12-05T05:16:55,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-05T05:16:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:55,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T05:16:55,473 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:16:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:55,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:55,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f1e5e1de38074cb19ab6e462afdc0f70_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375814458/Put/seqid=0 2024-12-05T05:16:55,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742007_1183 (size=12454) 2024-12-05T05:16:55,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:55,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:55,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375875622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,630 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375875626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375875627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,631 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375875733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375875733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375875733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,745 INFO [master/8edcaf2ecc84:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T05:16:55,745 INFO [master/8edcaf2ecc84:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T05:16:55,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T05:16:55,857 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/cbf1476667f749f2b0f05f01d53cb101 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/cbf1476667f749f2b0f05f01d53cb101 2024-12-05T05:16:55,866 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#A#compaction#150 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:55,866 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/B of 09d09c462e929eda7f34ea9ccdd56981 into cbf1476667f749f2b0f05f01d53cb101(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:55,866 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:55,866 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/B, priority=13, startTime=1733375815419; duration=0sec 2024-12-05T05:16:55,867 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:55,867 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:B 2024-12-05T05:16:55,867 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/005f0e4d0feb444aa6b48707b163f750 is 175, key is test_row_0/A:col10/1733375814419/Put/seqid=0 2024-12-05T05:16:55,867 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:16:55,870 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36963 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:16:55,870 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/C is initiating minor compaction (all files) 2024-12-05T05:16:55,870 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/C in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:55,870 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/df44226e9f314a5e9ca004e48940445c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/bbe3b02063284191a5097d9cfad34d2d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/61b97f69722249819d6ab63327c942bb] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=36.1 K 2024-12-05T05:16:55,870 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting df44226e9f314a5e9ca004e48940445c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1733375811340 2024-12-05T05:16:55,871 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting bbe3b02063284191a5097d9cfad34d2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1733375812499 2024-12-05T05:16:55,872 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b97f69722249819d6ab63327c942bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375813772 2024-12-05T05:16:55,882 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#C#compaction#153 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:55,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742008_1184 (size=31717) 2024-12-05T05:16:55,883 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/51f8551757534c3bb6e95b2b4648a2a9 is 50, key is test_row_0/C:col10/1733375814419/Put/seqid=0 2024-12-05T05:16:55,889 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/005f0e4d0feb444aa6b48707b163f750 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/005f0e4d0feb444aa6b48707b163f750 2024-12-05T05:16:55,896 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/A of 09d09c462e929eda7f34ea9ccdd56981 into 005f0e4d0feb444aa6b48707b163f750(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:55,896 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:55,896 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/A, priority=13, startTime=1733375815418; duration=0sec 2024-12-05T05:16:55,896 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:55,896 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:A 2024-12-05T05:16:55,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:55,903 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f1e5e1de38074cb19ab6e462afdc0f70_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f1e5e1de38074cb19ab6e462afdc0f70_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:55,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/6da3eacb245241d78649fcdc3e560660, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:55,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/6da3eacb245241d78649fcdc3e560660 is 175, key is test_row_0/A:col10/1733375814458/Put/seqid=0 2024-12-05T05:16:55,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742009_1185 (size=12763) 2024-12-05T05:16:55,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742010_1186 (size=31255) 2024-12-05T05:16:55,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375875943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375875944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:55,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375875944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:56,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375876253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:56,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375876253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:56,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375876253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T05:16:56,314 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/6da3eacb245241d78649fcdc3e560660 2024-12-05T05:16:56,322 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/51f8551757534c3bb6e95b2b4648a2a9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/51f8551757534c3bb6e95b2b4648a2a9 2024-12-05T05:16:56,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/2d23b4600da94ee0b1c267b8e2b979e0 is 50, key is test_row_0/B:col10/1733375814458/Put/seqid=0 2024-12-05T05:16:56,333 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/C of 09d09c462e929eda7f34ea9ccdd56981 into 51f8551757534c3bb6e95b2b4648a2a9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:56,333 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:56,333 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/C, priority=13, startTime=1733375815419; duration=0sec 2024-12-05T05:16:56,333 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:56,333 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:C 2024-12-05T05:16:56,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742011_1187 (size=12301) 2024-12-05T05:16:56,733 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/2d23b4600da94ee0b1c267b8e2b979e0 2024-12-05T05:16:56,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/99538fc4a18e4607839779995362eb76 is 50, key is test_row_0/C:col10/1733375814458/Put/seqid=0 2024-12-05T05:16:56,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742012_1188 (size=12301) 2024-12-05T05:16:56,753 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/99538fc4a18e4607839779995362eb76 2024-12-05T05:16:56,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/6da3eacb245241d78649fcdc3e560660 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6da3eacb245241d78649fcdc3e560660 2024-12-05T05:16:56,763 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6da3eacb245241d78649fcdc3e560660, entries=150, sequenceid=278, filesize=30.5 K 2024-12-05T05:16:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/2d23b4600da94ee0b1c267b8e2b979e0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2d23b4600da94ee0b1c267b8e2b979e0 2024-12-05T05:16:56,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375876763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:56,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375876763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:56,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,769 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2d23b4600da94ee0b1c267b8e2b979e0, entries=150, sequenceid=278, filesize=12.0 K 2024-12-05T05:16:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375876763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:56,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:56,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/99538fc4a18e4607839779995362eb76 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/99538fc4a18e4607839779995362eb76 2024-12-05T05:16:56,775 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/99538fc4a18e4607839779995362eb76, entries=150, sequenceid=278, filesize=12.0 K 2024-12-05T05:16:56,777 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 09d09c462e929eda7f34ea9ccdd56981 in 1304ms, sequenceid=278, compaction requested=false 2024-12-05T05:16:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:56,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-05T05:16:56,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-05T05:16:56,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-05T05:16:56,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6260 sec 2024-12-05T05:16:56,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.6330 sec 2024-12-05T05:16:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-05T05:16:57,303 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:57,304 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-05T05:16:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T05:16:57,306 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:57,307 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:57,308 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T05:16:57,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-05T05:16:57,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:57,461 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:16:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:57,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205903109d802c84922ba4e6af858419274_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375815620/Put/seqid=0 2024-12-05T05:16:57,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742013_1189 (size=12454) 2024-12-05T05:16:57,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T05:16:57,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:57,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375877824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:57,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375877835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:57,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375877837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:57,894 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205903109d802c84922ba4e6af858419274_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205903109d802c84922ba4e6af858419274_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:57,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/f7c800b5e99c4bba815a51cb555f0e08, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:57,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/f7c800b5e99c4bba815a51cb555f0e08 is 175, key is test_row_0/A:col10/1733375815620/Put/seqid=0 2024-12-05T05:16:57,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742014_1190 (size=31255) 2024-12-05T05:16:57,914 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/f7c800b5e99c4bba815a51cb555f0e08 2024-12-05T05:16:57,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/e70b8d74a8ab483d9fb595beffd23e49 is 50, key is test_row_0/B:col10/1733375815620/Put/seqid=0 2024-12-05T05:16:57,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T05:16:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742015_1191 (size=12301) 2024-12-05T05:16:57,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375877942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375877942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:57,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:57,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375877953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:57,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:58,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375878152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:58,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375878152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:58,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:58,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375878163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:58,343 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/e70b8d74a8ab483d9fb595beffd23e49 2024-12-05T05:16:58,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/452356bdaeaa49988a35a82f2a572692 is 50, key is test_row_0/C:col10/1733375815620/Put/seqid=0 2024-12-05T05:16:58,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742016_1192 (size=12301) 2024-12-05T05:16:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T05:16:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:58,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:58,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375878463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:58,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375878463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:58,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:58,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375878474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:58,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:58,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:58,791 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/452356bdaeaa49988a35a82f2a572692 2024-12-05T05:16:58,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/f7c800b5e99c4bba815a51cb555f0e08 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/f7c800b5e99c4bba815a51cb555f0e08 2024-12-05T05:16:58,802 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/f7c800b5e99c4bba815a51cb555f0e08, entries=150, sequenceid=294, filesize=30.5 K 2024-12-05T05:16:58,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/e70b8d74a8ab483d9fb595beffd23e49 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/e70b8d74a8ab483d9fb595beffd23e49 2024-12-05T05:16:58,813 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/e70b8d74a8ab483d9fb595beffd23e49, entries=150, sequenceid=294, filesize=12.0 K 2024-12-05T05:16:58,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/452356bdaeaa49988a35a82f2a572692 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/452356bdaeaa49988a35a82f2a572692 2024-12-05T05:16:58,818 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/452356bdaeaa49988a35a82f2a572692, entries=150, sequenceid=294, filesize=12.0 K 2024-12-05T05:16:58,819 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 09d09c462e929eda7f34ea9ccdd56981 in 1359ms, sequenceid=294, compaction requested=true 2024-12-05T05:16:58,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:58,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:58,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-05T05:16:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-05T05:16:58,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-05T05:16:58,824 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5140 sec 2024-12-05T05:16:58,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5210 sec 2024-12-05T05:16:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:58,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-12-05T05:16:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:16:58,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054618eecb7fc742248aa1b225b29682a9_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375818976/Put/seqid=0 2024-12-05T05:16:58,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742017_1193 (size=12454) 2024-12-05T05:16:58,990 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:58,994 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054618eecb7fc742248aa1b225b29682a9_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054618eecb7fc742248aa1b225b29682a9_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:58,995 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/d1fc4fa75c86443bbc863c182ae446f8, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:58,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/d1fc4fa75c86443bbc863c182ae446f8 is 175, key is test_row_0/A:col10/1733375818976/Put/seqid=0 2024-12-05T05:16:59,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375878992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375878992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742018_1194 (size=31255) 2024-12-05T05:16:59,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375878993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375879112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375879113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375879323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375879325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,402 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/d1fc4fa75c86443bbc863c182ae446f8 2024-12-05T05:16:59,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/26d110b874704dfabc9a55c807d24a75 is 50, key is test_row_0/B:col10/1733375818976/Put/seqid=0 2024-12-05T05:16:59,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742019_1195 (size=12301) 2024-12-05T05:16:59,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/26d110b874704dfabc9a55c807d24a75 2024-12-05T05:16:59,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-05T05:16:59,453 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:16:59,454 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:16:59,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-05T05:16:59,456 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:16:59,457 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:16:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T05:16:59,457 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:16:59,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/b1b38ba20d984aa6b54b58608e91e550 is 50, key is test_row_0/C:col10/1733375818976/Put/seqid=0 2024-12-05T05:16:59,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742020_1196 (size=12301) 2024-12-05T05:16:59,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/b1b38ba20d984aa6b54b58608e91e550 2024-12-05T05:16:59,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/d1fc4fa75c86443bbc863c182ae446f8 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/d1fc4fa75c86443bbc863c182ae446f8 2024-12-05T05:16:59,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/d1fc4fa75c86443bbc863c182ae446f8, entries=150, sequenceid=316, filesize=30.5 K 2024-12-05T05:16:59,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/26d110b874704dfabc9a55c807d24a75 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/26d110b874704dfabc9a55c807d24a75 2024-12-05T05:16:59,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/26d110b874704dfabc9a55c807d24a75, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T05:16:59,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/b1b38ba20d984aa6b54b58608e91e550 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/b1b38ba20d984aa6b54b58608e91e550 2024-12-05T05:16:59,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/b1b38ba20d984aa6b54b58608e91e550, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T05:16:59,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 09d09c462e929eda7f34ea9ccdd56981 in 557ms, sequenceid=316, compaction requested=true 2024-12-05T05:16:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:16:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:59,534 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:16:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:16:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:59,534 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:16:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:16:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:59,536 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49666 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:16:59,536 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125482 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:16:59,536 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/B is initiating minor compaction (all files) 2024-12-05T05:16:59,536 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/A is initiating minor compaction (all files) 2024-12-05T05:16:59,536 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/B in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:59,536 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/A in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:59,536 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/cbf1476667f749f2b0f05f01d53cb101, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2d23b4600da94ee0b1c267b8e2b979e0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/e70b8d74a8ab483d9fb595beffd23e49, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/26d110b874704dfabc9a55c807d24a75] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=48.5 K 2024-12-05T05:16:59,536 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/005f0e4d0feb444aa6b48707b163f750, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6da3eacb245241d78649fcdc3e560660, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/f7c800b5e99c4bba815a51cb555f0e08, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/d1fc4fa75c86443bbc863c182ae446f8] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=122.5 K 2024-12-05T05:16:59,536 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:59,537 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/005f0e4d0feb444aa6b48707b163f750, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6da3eacb245241d78649fcdc3e560660, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/f7c800b5e99c4bba815a51cb555f0e08, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/d1fc4fa75c86443bbc863c182ae446f8] 2024-12-05T05:16:59,537 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbf1476667f749f2b0f05f01d53cb101, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375813772 2024-12-05T05:16:59,538 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 005f0e4d0feb444aa6b48707b163f750, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375813772 2024-12-05T05:16:59,538 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d23b4600da94ee0b1c267b8e2b979e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733375814458 2024-12-05T05:16:59,538 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da3eacb245241d78649fcdc3e560660, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733375814458 2024-12-05T05:16:59,538 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e70b8d74a8ab483d9fb595beffd23e49, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733375815620 2024-12-05T05:16:59,539 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26d110b874704dfabc9a55c807d24a75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375817832 2024-12-05T05:16:59,539 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f7c800b5e99c4bba815a51cb555f0e08, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733375815620 2024-12-05T05:16:59,539 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d1fc4fa75c86443bbc863c182ae446f8, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375817832 2024-12-05T05:16:59,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T05:16:59,565 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:59,570 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#B#compaction#163 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:59,571 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/272d19dd07824d60966b21dfaf27f3a4 is 50, key is test_row_0/B:col10/1733375818976/Put/seqid=0 2024-12-05T05:16:59,585 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205e4d7e8f9c4b346f5bb670c1a7a551563_09d09c462e929eda7f34ea9ccdd56981 store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:59,587 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205e4d7e8f9c4b346f5bb670c1a7a551563_09d09c462e929eda7f34ea9ccdd56981, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:59,588 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e4d7e8f9c4b346f5bb670c1a7a551563_09d09c462e929eda7f34ea9ccdd56981 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:59,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-05T05:16:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:59,612 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:16:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:16:59,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:16:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:16:59,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:16:59,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742022_1198 (size=4469) 2024-12-05T05:16:59,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742021_1197 (size=12949) 2024-12-05T05:16:59,623 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#A#compaction#162 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:59,623 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/7eff5d678d154fc6a152e1eae66473f0 is 175, key is test_row_0/A:col10/1733375818976/Put/seqid=0 2024-12-05T05:16:59,628 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/272d19dd07824d60966b21dfaf27f3a4 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/272d19dd07824d60966b21dfaf27f3a4 2024-12-05T05:16:59,636 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/B of 09d09c462e929eda7f34ea9ccdd56981 into 272d19dd07824d60966b21dfaf27f3a4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:59,637 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:59,637 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/B, priority=12, startTime=1733375819534; duration=0sec 2024-12-05T05:16:59,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:16:59,637 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:16:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:59,637 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:B 2024-12-05T05:16:59,637 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:16:59,638 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49666 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:16:59,639 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/C is initiating minor compaction (all files) 2024-12-05T05:16:59,639 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/C in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:16:59,639 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/51f8551757534c3bb6e95b2b4648a2a9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/99538fc4a18e4607839779995362eb76, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/452356bdaeaa49988a35a82f2a572692, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/b1b38ba20d984aa6b54b58608e91e550] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=48.5 K 2024-12-05T05:16:59,639 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51f8551757534c3bb6e95b2b4648a2a9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375813772 2024-12-05T05:16:59,640 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99538fc4a18e4607839779995362eb76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1733375814458 2024-12-05T05:16:59,640 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 452356bdaeaa49988a35a82f2a572692, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=294, earliestPutTs=1733375815620 2024-12-05T05:16:59,640 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1b38ba20d984aa6b54b58608e91e550, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375817832 2024-12-05T05:16:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205be46ec04ee8a45708cc80ff37af749c8_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375818992/Put/seqid=0 2024-12-05T05:16:59,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742023_1199 (size=31903) 2024-12-05T05:16:59,687 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/7eff5d678d154fc6a152e1eae66473f0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/7eff5d678d154fc6a152e1eae66473f0 2024-12-05T05:16:59,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733375819669, max timestamp=1733375819687Num uniques 859; 2024-12-05T05:16:59,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375819669, max timestamp=1733375819687Num uniques 859; 2024-12-05T05:16:59,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:59,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:59,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375819669, max timestamp=1733375819687Num uniques 859; 2024-12-05T05:16:59,694 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:16:59,699 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#C#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:16:59,699 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/6939bca485a64e77a1a5c0d229912a29 is 50, key is test_row_0/C:col10/1733375818976/Put/seqid=0 2024-12-05T05:16:59,701 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/A of 09d09c462e929eda7f34ea9ccdd56981 into 7eff5d678d154fc6a152e1eae66473f0(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:59,702 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:59,702 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/A, priority=12, startTime=1733375819534; duration=0sec 2024-12-05T05:16:59,702 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:59,702 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:A 2024-12-05T05:16:59,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742024_1200 (size=12454) 2024-12-05T05:16:59,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:16:59,744 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205be46ec04ee8a45708cc80ff37af749c8_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205be46ec04ee8a45708cc80ff37af749c8_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:16:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/ff79772215474e7fb2954f516fcfdc7d, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:16:59,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/ff79772215474e7fb2954f516fcfdc7d is 175, key is test_row_0/A:col10/1733375818992/Put/seqid=0 2024-12-05T05:16:59,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742025_1201 (size=12949) 2024-12-05T05:16:59,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375879747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375879749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742026_1202 (size=31255) 2024-12-05T05:16:59,754 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=326, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/ff79772215474e7fb2954f516fcfdc7d 2024-12-05T05:16:59,758 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/6939bca485a64e77a1a5c0d229912a29 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/6939bca485a64e77a1a5c0d229912a29 2024-12-05T05:16:59,765 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/C of 09d09c462e929eda7f34ea9ccdd56981 into 6939bca485a64e77a1a5c0d229912a29(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:16:59,765 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:16:59,765 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/C, priority=12, startTime=1733375819534; duration=0sec 2024-12-05T05:16:59,765 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:16:59,765 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:C 2024-12-05T05:16:59,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/451980f04aff4964a22e09e44e356c9e is 50, key is test_row_0/B:col10/1733375818992/Put/seqid=0 2024-12-05T05:16:59,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T05:16:59,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742027_1203 (size=12301) 2024-12-05T05:16:59,787 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/451980f04aff4964a22e09e44e356c9e 2024-12-05T05:16:59,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/90d3b311ee53441cad63a88d924cf29d is 50, key is test_row_0/C:col10/1733375818992/Put/seqid=0 2024-12-05T05:16:59,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742028_1204 (size=12301) 2024-12-05T05:16:59,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375879854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,859 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:16:59,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:16:59,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375879863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:16:59,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:16:59,868 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375880013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,014 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375880062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,065 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375880072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T05:17:00,209 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/90d3b311ee53441cad63a88d924cf29d 2024-12-05T05:17:00,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/ff79772215474e7fb2954f516fcfdc7d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ff79772215474e7fb2954f516fcfdc7d 2024-12-05T05:17:00,239 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ff79772215474e7fb2954f516fcfdc7d, entries=150, sequenceid=326, filesize=30.5 K 2024-12-05T05:17:00,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/451980f04aff4964a22e09e44e356c9e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/451980f04aff4964a22e09e44e356c9e 2024-12-05T05:17:00,252 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/451980f04aff4964a22e09e44e356c9e, entries=150, sequenceid=326, filesize=12.0 K 2024-12-05T05:17:00,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/90d3b311ee53441cad63a88d924cf29d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/90d3b311ee53441cad63a88d924cf29d 2024-12-05T05:17:00,258 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/90d3b311ee53441cad63a88d924cf29d, entries=150, sequenceid=326, filesize=12.0 K 2024-12-05T05:17:00,259 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 09d09c462e929eda7f34ea9ccdd56981 in 647ms, sequenceid=326, compaction requested=false 2024-12-05T05:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:00,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-05T05:17:00,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-05T05:17:00,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-05T05:17:00,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 805 msec 2024-12-05T05:17:00,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 811 msec 2024-12-05T05:17:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:00,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-05T05:17:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:17:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:17:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:17:00,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:00,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205685d3ce2780644eba6268063b29fd985_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375819746/Put/seqid=0 2024-12-05T05:17:00,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375880401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742029_1205 (size=14994) 2024-12-05T05:17:00,407 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:00,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375880405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,411 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,415 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205685d3ce2780644eba6268063b29fd985_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205685d3ce2780644eba6268063b29fd985_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:00,416 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:00,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978 is 175, key is test_row_0/A:col10/1733375819746/Put/seqid=0 2024-12-05T05:17:00,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742030_1206 (size=39949) 2024-12-05T05:17:00,449 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978 2024-12-05T05:17:00,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/d093a984b0b74111a969b8aea17254ec is 50, key is test_row_0/B:col10/1733375819746/Put/seqid=0 2024-12-05T05:17:00,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742031_1207 (size=12301) 2024-12-05T05:17:00,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375880512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375880522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-05T05:17:00,593 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:00,593 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-05T05:17:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T05:17:00,596 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:00,597 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:00,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:00,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T05:17:00,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375880722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:00,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375880733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:00,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:00,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T05:17:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:17:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:00,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:00,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:00,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:00,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/d093a984b0b74111a969b8aea17254ec 2024-12-05T05:17:00,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/a6baec13706e462da690177179bd14da is 50, key is test_row_0/C:col10/1733375819746/Put/seqid=0 2024-12-05T05:17:00,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T05:17:00,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:00,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:17:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:00,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:00,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:00,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742032_1208 (size=12301) 2024-12-05T05:17:00,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/a6baec13706e462da690177179bd14da 2024-12-05T05:17:00,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T05:17:00,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978 2024-12-05T05:17:00,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978, entries=200, sequenceid=366, filesize=39.0 K 2024-12-05T05:17:00,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/d093a984b0b74111a969b8aea17254ec as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/d093a984b0b74111a969b8aea17254ec 2024-12-05T05:17:00,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/d093a984b0b74111a969b8aea17254ec, entries=150, sequenceid=366, filesize=12.0 K 2024-12-05T05:17:00,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/a6baec13706e462da690177179bd14da as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/a6baec13706e462da690177179bd14da 2024-12-05T05:17:00,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/a6baec13706e462da690177179bd14da, entries=150, sequenceid=366, filesize=12.0 K 2024-12-05T05:17:00,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 09d09c462e929eda7f34ea9ccdd56981 in 570ms, sequenceid=366, compaction requested=true 2024-12-05T05:17:00,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:00,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:00,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:00,947 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:00,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:00,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:00,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:00,947 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T05:17:00,947 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:00,948 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:00,948 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/B is initiating minor compaction (all files) 2024-12-05T05:17:00,948 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/B in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:00,949 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/272d19dd07824d60966b21dfaf27f3a4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/451980f04aff4964a22e09e44e356c9e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/d093a984b0b74111a969b8aea17254ec] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=36.7 K 2024-12-05T05:17:00,949 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:00,949 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/A is initiating minor compaction (all files) 2024-12-05T05:17:00,949 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/A in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:00,949 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/7eff5d678d154fc6a152e1eae66473f0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ff79772215474e7fb2954f516fcfdc7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=100.7 K 2024-12-05T05:17:00,949 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:00,949 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/7eff5d678d154fc6a152e1eae66473f0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ff79772215474e7fb2954f516fcfdc7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978] 2024-12-05T05:17:00,950 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 272d19dd07824d60966b21dfaf27f3a4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375817832 2024-12-05T05:17:00,950 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 451980f04aff4964a22e09e44e356c9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733375818986 2024-12-05T05:17:00,950 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7eff5d678d154fc6a152e1eae66473f0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375817832 2024-12-05T05:17:00,951 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff79772215474e7fb2954f516fcfdc7d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733375818986 2024-12-05T05:17:00,951 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d093a984b0b74111a969b8aea17254ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733375819719 2024-12-05T05:17:00,951 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16a7ee4b48ab4ff6ae7cdf03c2d7c978, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733375819719 2024-12-05T05:17:00,986 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:00,987 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/2f29633b3a5c4f52905cdc7f19ae5d0b is 50, key is test_row_0/B:col10/1733375819746/Put/seqid=0 2024-12-05T05:17:00,988 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:00,990 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205eab9e65e858848a2acda785efe30edde_09d09c462e929eda7f34ea9ccdd56981 store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:00,992 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205eab9e65e858848a2acda785efe30edde_09d09c462e929eda7f34ea9ccdd56981, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:00,992 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205eab9e65e858848a2acda785efe30edde_09d09c462e929eda7f34ea9ccdd56981 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:00,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742034_1210 (size=4469) 2024-12-05T05:17:00,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742033_1209 (size=13051) 2024-12-05T05:17:01,001 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#A#compaction#172 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:01,001 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/edb9a136806e482d888f32f90e3cbfda is 175, key is test_row_0/A:col10/1733375819746/Put/seqid=0 2024-12-05T05:17:01,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742035_1211 (size=32005) 2024-12-05T05:17:01,027 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/edb9a136806e482d888f32f90e3cbfda as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/edb9a136806e482d888f32f90e3cbfda 2024-12-05T05:17:01,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:01,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:17:01,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:17:01,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:01,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:17:01,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:01,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:17:01,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:01,040 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/A of 09d09c462e929eda7f34ea9ccdd56981 into edb9a136806e482d888f32f90e3cbfda(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:01,040 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:01,040 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/A, priority=13, startTime=1733375820947; duration=0sec 2024-12-05T05:17:01,041 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:01,041 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:A 2024-12-05T05:17:01,041 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:01,043 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:01,043 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/C is initiating minor compaction (all files) 2024-12-05T05:17:01,043 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/C in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,044 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/6939bca485a64e77a1a5c0d229912a29, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/90d3b311ee53441cad63a88d924cf29d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/a6baec13706e462da690177179bd14da] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=36.7 K 2024-12-05T05:17:01,044 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6939bca485a64e77a1a5c0d229912a29, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375817832 2024-12-05T05:17:01,047 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d3b311ee53441cad63a88d924cf29d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1733375818986 2024-12-05T05:17:01,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d1de19b2e52a498da712ecc6987dd3b6_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375820387/Put/seqid=0 2024-12-05T05:17:01,048 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6baec13706e462da690177179bd14da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733375819719 2024-12-05T05:17:01,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T05:17:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:17:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742036_1212 (size=12454) 2024-12-05T05:17:01,067 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:01,070 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:01,070 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/1868824a44af4364bf1a62336437e663 is 50, key is test_row_0/C:col10/1733375819746/Put/seqid=0 2024-12-05T05:17:01,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742037_1213 (size=13051) 2024-12-05T05:17:01,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d1de19b2e52a498da712ecc6987dd3b6_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d1de19b2e52a498da712ecc6987dd3b6_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:01,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/c1f3203b6f664efaa069c161650292a3, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:01,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/c1f3203b6f664efaa069c161650292a3 is 175, key is test_row_0/A:col10/1733375820387/Put/seqid=0 2024-12-05T05:17:01,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742038_1214 (size=31255) 2024-12-05T05:17:01,083 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=377, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/c1f3203b6f664efaa069c161650292a3 2024-12-05T05:17:01,083 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/1868824a44af4364bf1a62336437e663 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1868824a44af4364bf1a62336437e663 2024-12-05T05:17:01,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/5f5a3c4a9fc84a7f8fdce1da4871604b is 50, key is test_row_0/B:col10/1733375820387/Put/seqid=0 2024-12-05T05:17:01,093 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/C of 09d09c462e929eda7f34ea9ccdd56981 into 1868824a44af4364bf1a62336437e663(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:01,093 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:01,093 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/C, priority=13, startTime=1733375820947; duration=0sec 2024-12-05T05:17:01,094 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:01,094 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:C 2024-12-05T05:17:01,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733375821059, max timestamp=1733375821102Num uniques 859; 2024-12-05T05:17:01,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-05T05:17:01,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733375821059, max timestamp=1733375821102Num uniques 859; 2024-12-05T05:17:01,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-05T05:17:01,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733375821059, max timestamp=1733375821102Num uniques 859; 2024-12-05T05:17:01,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-05T05:17:01,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742039_1215 (size=12301) 2024-12-05T05:17:01,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375881156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375881158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,161 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,162 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T05:17:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:17:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T05:17:01,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375881273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375881273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T05:17:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:17:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,404 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/2f29633b3a5c4f52905cdc7f19ae5d0b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2f29633b3a5c4f52905cdc7f19ae5d0b 2024-12-05T05:17:01,413 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/B of 09d09c462e929eda7f34ea9ccdd56981 into 2f29633b3a5c4f52905cdc7f19ae5d0b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:01,413 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:01,413 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/B, priority=13, startTime=1733375820947; duration=0sec 2024-12-05T05:17:01,413 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:01,413 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:B 2024-12-05T05:17:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375881482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375881482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T05:17:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:17:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:01,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/5f5a3c4a9fc84a7f8fdce1da4871604b 2024-12-05T05:17:01,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/ca1902d54cf2460b9763672430fcf5b6 is 50, key is test_row_0/C:col10/1733375820387/Put/seqid=0 2024-12-05T05:17:01,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742040_1216 (size=12301) 2024-12-05T05:17:01,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/ca1902d54cf2460b9763672430fcf5b6 2024-12-05T05:17:01,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/c1f3203b6f664efaa069c161650292a3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1f3203b6f664efaa069c161650292a3 2024-12-05T05:17:01,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1f3203b6f664efaa069c161650292a3, entries=150, sequenceid=377, filesize=30.5 K 2024-12-05T05:17:01,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/5f5a3c4a9fc84a7f8fdce1da4871604b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5f5a3c4a9fc84a7f8fdce1da4871604b 2024-12-05T05:17:01,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5f5a3c4a9fc84a7f8fdce1da4871604b, entries=150, sequenceid=377, filesize=12.0 K 2024-12-05T05:17:01,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/ca1902d54cf2460b9763672430fcf5b6 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ca1902d54cf2460b9763672430fcf5b6 2024-12-05T05:17:01,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ca1902d54cf2460b9763672430fcf5b6, entries=150, sequenceid=377, filesize=12.0 K 2024-12-05T05:17:01,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 09d09c462e929eda7f34ea9ccdd56981 in 576ms, sequenceid=377, compaction requested=false 2024-12-05T05:17:01,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:01,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-05T05:17:01,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:01,672 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-12-05T05:17:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:17:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:17:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:17:01,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:01,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052ad13ca8036c4f4baea524d2002ca8e7_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375821155/Put/seqid=0 2024-12-05T05:17:01,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742041_1217 (size=12454) 2024-12-05T05:17:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T05:17:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:01,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. as already flushing 2024-12-05T05:17:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375881802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375881802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375881913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:01,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375881913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:02,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43978 deadline: 1733375882032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:02,033 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:02,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:02,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:02,097 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052ad13ca8036c4f4baea524d2002ca8e7_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ad13ca8036c4f4baea524d2002ca8e7_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:02,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/579a376e83ab4780b066cba48d7dfd58, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:02,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/579a376e83ab4780b066cba48d7dfd58 is 175, key is test_row_0/A:col10/1733375821155/Put/seqid=0 2024-12-05T05:17:02,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742042_1218 (size=31255) 2024-12-05T05:17:02,120 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/579a376e83ab4780b066cba48d7dfd58 2024-12-05T05:17:02,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375882123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:02,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375882132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:02,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,141 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:02,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/9e437f46ad5145f6b267450a079b923d is 50, key is test_row_0/B:col10/1733375821155/Put/seqid=0 2024-12-05T05:17:02,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742043_1219 (size=12301) 2024-12-05T05:17:02,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375882433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:02,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375882452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:02,559 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/9e437f46ad5145f6b267450a079b923d 2024-12-05T05:17:02,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/8ff7847586e5438d9f879dda52b5e1d5 is 50, key is test_row_0/C:col10/1733375821155/Put/seqid=0 2024-12-05T05:17:02,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742044_1220 (size=12301) 2024-12-05T05:17:02,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T05:17:02,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:02,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375882943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:02,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375882962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:02,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:02,980 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/8ff7847586e5438d9f879dda52b5e1d5 2024-12-05T05:17:02,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/579a376e83ab4780b066cba48d7dfd58 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/579a376e83ab4780b066cba48d7dfd58 2024-12-05T05:17:02,989 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/579a376e83ab4780b066cba48d7dfd58, entries=150, sequenceid=413, filesize=30.5 K 2024-12-05T05:17:02,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/9e437f46ad5145f6b267450a079b923d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9e437f46ad5145f6b267450a079b923d 2024-12-05T05:17:02,994 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9e437f46ad5145f6b267450a079b923d, entries=150, sequenceid=413, filesize=12.0 K 2024-12-05T05:17:02,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/8ff7847586e5438d9f879dda52b5e1d5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/8ff7847586e5438d9f879dda52b5e1d5 2024-12-05T05:17:02,999 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/8ff7847586e5438d9f879dda52b5e1d5, entries=150, sequenceid=413, filesize=12.0 K 2024-12-05T05:17:03,000 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 09d09c462e929eda7f34ea9ccdd56981 in 1328ms, sequenceid=413, compaction requested=true 2024-12-05T05:17:03,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:03,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:03,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-05T05:17:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-05T05:17:03,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-05T05:17:03,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4040 sec 2024-12-05T05:17:03,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.4100 sec 2024-12-05T05:17:03,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:03,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:17:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:17:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:17:03,134 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:17:03,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:03,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054df908453c87453aa7cb5f63ed84b3ab_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375823133/Put/seqid=0 2024-12-05T05:17:03,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742045_1221 (size=12454) 2024-12-05T05:17:03,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:03,155 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054df908453c87453aa7cb5f63ed84b3ab_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054df908453c87453aa7cb5f63ed84b3ab_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:03,157 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/5086c50391c0431798f2af091adbf772, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:03,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/5086c50391c0431798f2af091adbf772 is 175, key is test_row_0/A:col10/1733375823133/Put/seqid=0 2024-12-05T05:17:03,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742046_1222 (size=31255) 2024-12-05T05:17:03,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375823168, max timestamp=1733375823184Num uniques 859; 2024-12-05T05:17:03,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:03,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375823168, max timestamp=1733375823184Num uniques 859; 2024-12-05T05:17:03,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:03,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375823168, max timestamp=1733375823184Num uniques 859; 2024-12-05T05:17:03,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:03,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375883222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375883223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375883333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375883333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375883542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375883542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,562 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=424, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/5086c50391c0431798f2af091adbf772 2024-12-05T05:17:03,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/67829e132a9b4cd78ab3a2da212bc569 is 50, key is test_row_0/B:col10/1733375823133/Put/seqid=0 2024-12-05T05:17:03,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742047_1223 (size=12301) 2024-12-05T05:17:03,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375883853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375883853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43962 deadline: 1733375883953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,957 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:03,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/67829e132a9b4cd78ab3a2da212bc569 2024-12-05T05:17:03,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/de483746d63c47c8852bd07b64b40fa9 is 50, key is test_row_0/C:col10/1733375823133/Put/seqid=0 2024-12-05T05:17:03,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742048_1224 (size=12301) 2024-12-05T05:17:03,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43960 deadline: 1733375883983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:03,987 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 09d09c462e929eda7f34ea9ccdd56981 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:04,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:04,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43972 deadline: 1733375884363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:04,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43946 deadline: 1733375884363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:04,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:04,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/de483746d63c47c8852bd07b64b40fa9 2024-12-05T05:17:04,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/5086c50391c0431798f2af091adbf772 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/5086c50391c0431798f2af091adbf772 2024-12-05T05:17:04,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/5086c50391c0431798f2af091adbf772, entries=150, sequenceid=424, filesize=30.5 K 2024-12-05T05:17:04,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/67829e132a9b4cd78ab3a2da212bc569 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/67829e132a9b4cd78ab3a2da212bc569 2024-12-05T05:17:04,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/67829e132a9b4cd78ab3a2da212bc569, entries=150, sequenceid=424, filesize=12.0 K 2024-12-05T05:17:04,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/de483746d63c47c8852bd07b64b40fa9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/de483746d63c47c8852bd07b64b40fa9 2024-12-05T05:17:04,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/de483746d63c47c8852bd07b64b40fa9, entries=150, sequenceid=424, filesize=12.0 K 2024-12-05T05:17:04,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 09d09c462e929eda7f34ea9ccdd56981 in 1287ms, sequenceid=424, compaction requested=true 2024-12-05T05:17:04,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:04,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:04,421 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:04,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:04,421 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:04,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:04,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:04,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09d09c462e929eda7f34ea9ccdd56981:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:04,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:04,427 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:04,427 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/B is initiating minor compaction (all files) 2024-12-05T05:17:04,427 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/B in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:04,427 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2f29633b3a5c4f52905cdc7f19ae5d0b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5f5a3c4a9fc84a7f8fdce1da4871604b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9e437f46ad5145f6b267450a079b923d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/67829e132a9b4cd78ab3a2da212bc569] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=48.8 K 2024-12-05T05:17:04,428 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125770 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:04,428 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/A is initiating minor compaction (all files) 2024-12-05T05:17:04,428 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/A in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:04,428 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/edb9a136806e482d888f32f90e3cbfda, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1f3203b6f664efaa069c161650292a3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/579a376e83ab4780b066cba48d7dfd58, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/5086c50391c0431798f2af091adbf772] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=122.8 K 2024-12-05T05:17:04,428 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:04,428 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/edb9a136806e482d888f32f90e3cbfda, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1f3203b6f664efaa069c161650292a3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/579a376e83ab4780b066cba48d7dfd58, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/5086c50391c0431798f2af091adbf772] 2024-12-05T05:17:04,428 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f29633b3a5c4f52905cdc7f19ae5d0b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733375819719 2024-12-05T05:17:04,429 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting edb9a136806e482d888f32f90e3cbfda, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733375819719 2024-12-05T05:17:04,429 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f5a3c4a9fc84a7f8fdce1da4871604b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733375820387 2024-12-05T05:17:04,429 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1f3203b6f664efaa069c161650292a3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733375820387 2024-12-05T05:17:04,429 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 579a376e83ab4780b066cba48d7dfd58, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733375821133 2024-12-05T05:17:04,429 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e437f46ad5145f6b267450a079b923d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733375821133 2024-12-05T05:17:04,430 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 67829e132a9b4cd78ab3a2da212bc569, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733375821799 2024-12-05T05:17:04,430 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5086c50391c0431798f2af091adbf772, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733375821799 2024-12-05T05:17:04,459 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:04,466 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412057df561aed82e424c99a922588e9aed76_09d09c462e929eda7f34ea9ccdd56981 store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:04,466 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:04,467 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/24bd27062dba45efb3a958d9126eb3cb is 50, key is test_row_0/B:col10/1733375823133/Put/seqid=0 2024-12-05T05:17:04,468 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412057df561aed82e424c99a922588e9aed76_09d09c462e929eda7f34ea9ccdd56981, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:04,468 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412057df561aed82e424c99a922588e9aed76_09d09c462e929eda7f34ea9ccdd56981 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:04,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742049_1225 (size=13187) 2024-12-05T05:17:04,505 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/24bd27062dba45efb3a958d9126eb3cb as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/24bd27062dba45efb3a958d9126eb3cb 2024-12-05T05:17:04,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742050_1226 (size=4469) 2024-12-05T05:17:04,508 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#A#compaction#183 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:04,509 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/dc0c112fccf445fda1634548dda1d88e is 175, key is test_row_0/A:col10/1733375823133/Put/seqid=0 2024-12-05T05:17:04,512 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/B of 09d09c462e929eda7f34ea9ccdd56981 into 24bd27062dba45efb3a958d9126eb3cb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:04,512 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:04,512 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/B, priority=12, startTime=1733375824421; duration=0sec 2024-12-05T05:17:04,512 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:04,512 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:B 2024-12-05T05:17:04,512 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:04,514 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:04,514 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 09d09c462e929eda7f34ea9ccdd56981/C is initiating minor compaction (all files) 2024-12-05T05:17:04,514 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09d09c462e929eda7f34ea9ccdd56981/C in TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:04,514 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1868824a44af4364bf1a62336437e663, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ca1902d54cf2460b9763672430fcf5b6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/8ff7847586e5438d9f879dda52b5e1d5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/de483746d63c47c8852bd07b64b40fa9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp, totalSize=48.8 K 2024-12-05T05:17:04,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742051_1227 (size=32141) 2024-12-05T05:17:04,516 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 1868824a44af4364bf1a62336437e663, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1733375819719 2024-12-05T05:17:04,517 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ca1902d54cf2460b9763672430fcf5b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1733375820387 2024-12-05T05:17:04,518 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ff7847586e5438d9f879dda52b5e1d5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1733375821133 2024-12-05T05:17:04,519 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting de483746d63c47c8852bd07b64b40fa9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1733375821799 2024-12-05T05:17:04,524 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/dc0c112fccf445fda1634548dda1d88e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/dc0c112fccf445fda1634548dda1d88e 2024-12-05T05:17:04,531 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/A of 09d09c462e929eda7f34ea9ccdd56981 into dc0c112fccf445fda1634548dda1d88e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:04,531 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:04,531 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/A, priority=12, startTime=1733375824421; duration=0sec 2024-12-05T05:17:04,531 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:04,531 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:A 2024-12-05T05:17:04,532 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09d09c462e929eda7f34ea9ccdd56981#C#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:04,532 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/4c9dd798a87a44f5a474524e8decec24 is 50, key is test_row_0/C:col10/1733375823133/Put/seqid=0 2024-12-05T05:17:04,535 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-12-05T05:17:04,535 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:04,535 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:04,536 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:04,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:04,538 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-12-05T05:17:04,538 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:04,538 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:04,538 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:04,538 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:04,539 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-12-05T05:17:04,539 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:04,539 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:04,539 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:04,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:04,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742052_1228 (size=13187) 2024-12-05T05:17:04,547 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-12-05T05:17:04,547 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:04,547 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:04,548 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:04,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:04,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-05T05:17:04,764 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:04,948 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/4c9dd798a87a44f5a474524e8decec24 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/4c9dd798a87a44f5a474524e8decec24 2024-12-05T05:17:04,954 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09d09c462e929eda7f34ea9ccdd56981/C of 09d09c462e929eda7f34ea9ccdd56981 into 4c9dd798a87a44f5a474524e8decec24(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:04,954 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:04,954 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., storeName=09d09c462e929eda7f34ea9ccdd56981/C, priority=12, startTime=1733375824422; duration=0sec 2024-12-05T05:17:04,954 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:04,954 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09d09c462e929eda7f34ea9ccdd56981:C 2024-12-05T05:17:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:05,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T05:17:05,374 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-12-05T05:17:05,374 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:05,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:17:05,374 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:05,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:05,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:17:05,374 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:05,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:05,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:05,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:17:05,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:05,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f3217286b6c14071930ac5be50d28c48_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375825373/Put/seqid=0 2024-12-05T05:17:05,384 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-12-05T05:17:05,384 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:05,384 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:05,385 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:05,385 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:05,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742053_1229 (size=12454) 2024-12-05T05:17:05,790 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:05,797 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f3217286b6c14071930ac5be50d28c48_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f3217286b6c14071930ac5be50d28c48_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:05,798 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/587081a430d346f69bb1013e72e79841, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:05,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/587081a430d346f69bb1013e72e79841 is 175, key is test_row_0/A:col10/1733375825373/Put/seqid=0 2024-12-05T05:17:05,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742054_1230 (size=31255) 2024-12-05T05:17:05,974 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-12-05T05:17:05,974 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:05,974 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:05,975 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:05,975 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:06,004 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-12-05T05:17:06,004 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:06,004 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:06,004 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:06,004 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:06,064 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-12-05T05:17:06,064 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:06,064 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:06,064 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:06,064 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 125 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2948 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2951 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1372 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4113 rows 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1369 2024-12-05T05:17:06,064 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4102 rows 2024-12-05T05:17:06,064 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T05:17:06,064 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:17:06,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:06,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:06,065 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:06,065 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T05:17:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T05:17:06,068 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375826068"}]},"ts":"1733375826068"} 2024-12-05T05:17:06,069 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T05:17:06,069 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T05:17:06,070 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:17:06,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, UNASSIGN}] 2024-12-05T05:17:06,072 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, UNASSIGN 2024-12-05T05:17:06,072 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=09d09c462e929eda7f34ea9ccdd56981, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:06,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, UNASSIGN because future has completed 2024-12-05T05:17:06,074 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:17:06,074 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:17:06,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T05:17:06,203 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=464, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/587081a430d346f69bb1013e72e79841 2024-12-05T05:17:06,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/621f5d222d164e8bafb826b81cf24c54 is 50, key is test_row_0/B:col10/1733375825373/Put/seqid=0 2024-12-05T05:17:06,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742055_1231 (size=12301) 2024-12-05T05:17:06,215 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/621f5d222d164e8bafb826b81cf24c54 2024-12-05T05:17:06,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/9232dec37f9e4850ad36036d533317f5 is 50, key is test_row_0/C:col10/1733375825373/Put/seqid=0 2024-12-05T05:17:06,226 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(122): Close 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:06,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742056_1232 (size=12301) 2024-12-05T05:17:06,226 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:17:06,226 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1722): Closing 09d09c462e929eda7f34ea9ccdd56981, disabling compactions & flushes 2024-12-05T05:17:06,226 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:06,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T05:17:06,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/9232dec37f9e4850ad36036d533317f5 2024-12-05T05:17:06,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/587081a430d346f69bb1013e72e79841 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/587081a430d346f69bb1013e72e79841 2024-12-05T05:17:06,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/587081a430d346f69bb1013e72e79841, entries=150, sequenceid=464, filesize=30.5 K 2024-12-05T05:17:06,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/621f5d222d164e8bafb826b81cf24c54 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/621f5d222d164e8bafb826b81cf24c54 2024-12-05T05:17:06,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/621f5d222d164e8bafb826b81cf24c54, entries=150, sequenceid=464, filesize=12.0 K 2024-12-05T05:17:06,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/9232dec37f9e4850ad36036d533317f5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/9232dec37f9e4850ad36036d533317f5 2024-12-05T05:17:06,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/9232dec37f9e4850ad36036d533317f5, entries=150, sequenceid=464, filesize=12.0 K 2024-12-05T05:17:06,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=26.84 KB/27480 for 09d09c462e929eda7f34ea9ccdd56981 in 1270ms, sequenceid=464, compaction requested=false 2024-12-05T05:17:06,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09d09c462e929eda7f34ea9ccdd56981: 2024-12-05T05:17:06,644 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. after waiting 0 ms 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:06,644 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(2902): Flushing 09d09c462e929eda7f34ea9ccdd56981 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=A 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=B 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09d09c462e929eda7f34ea9ccdd56981, store=C 2024-12-05T05:17:06,644 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:06,649 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052ec22024c51c415cb8320d2f659e9003_09d09c462e929eda7f34ea9ccdd56981 is 50, key is test_row_0/A:col10/1733375826062/Put/seqid=0 2024-12-05T05:17:06,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742057_1233 (size=12454) 2024-12-05T05:17:06,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T05:17:07,053 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:07,056 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412052ec22024c51c415cb8320d2f659e9003_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ec22024c51c415cb8320d2f659e9003_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:07,057 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/0d6c0b6c1950439a8beec2e1ba3e4bbe, store: [table=TestAcidGuarantees family=A region=09d09c462e929eda7f34ea9ccdd56981] 2024-12-05T05:17:07,058 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/0d6c0b6c1950439a8beec2e1ba3e4bbe is 175, key is test_row_0/A:col10/1733375826062/Put/seqid=0 2024-12-05T05:17:07,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742058_1234 (size=31255) 2024-12-05T05:17:07,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T05:17:07,462 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=471, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/0d6c0b6c1950439a8beec2e1ba3e4bbe 2024-12-05T05:17:07,469 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/0496ffea49874ec48c62638c85fc1b04 is 50, key is test_row_0/B:col10/1733375826062/Put/seqid=0 2024-12-05T05:17:07,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742059_1235 (size=12301) 2024-12-05T05:17:07,874 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/0496ffea49874ec48c62638c85fc1b04 2024-12-05T05:17:07,881 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/cce33f9d848743eb89e594fca9262719 is 50, key is test_row_0/C:col10/1733375826062/Put/seqid=0 2024-12-05T05:17:07,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742060_1236 (size=12301) 2024-12-05T05:17:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T05:17:08,286 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/cce33f9d848743eb89e594fca9262719 2024-12-05T05:17:08,290 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/A/0d6c0b6c1950439a8beec2e1ba3e4bbe as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/0d6c0b6c1950439a8beec2e1ba3e4bbe 2024-12-05T05:17:08,296 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/0d6c0b6c1950439a8beec2e1ba3e4bbe, entries=150, sequenceid=471, filesize=30.5 K 2024-12-05T05:17:08,297 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/B/0496ffea49874ec48c62638c85fc1b04 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0496ffea49874ec48c62638c85fc1b04 2024-12-05T05:17:08,301 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0496ffea49874ec48c62638c85fc1b04, entries=150, sequenceid=471, filesize=12.0 K 2024-12-05T05:17:08,302 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/.tmp/C/cce33f9d848743eb89e594fca9262719 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/cce33f9d848743eb89e594fca9262719 2024-12-05T05:17:08,306 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/cce33f9d848743eb89e594fca9262719, entries=150, sequenceid=471, filesize=12.0 K 2024-12-05T05:17:08,307 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 09d09c462e929eda7f34ea9ccdd56981 in 1663ms, sequenceid=471, compaction requested=true 2024-12-05T05:17:08,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/39aa317f5e894268927e92c8ebcf791a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/05f63486d53e4af98cbf94a777da4c7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b495cdada6344b7c8ebfe982cae48a2b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ac4c684f63074edd82e2a11e75ee0ff0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/fd95b2d88603489e99ba0a07b128d4da, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/55aa6cc29ec84401a08e2d2762b4e648, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/bb4e9704f20546ed8982d49070b48a54, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b70285c5bd9d43b9bf372d17a8fe6c2a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/96fcb07dfb704e4f98e2828343b36590, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/668ddaaedb0f48d19a1a6627a64fa986, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6aa88a569187444bae0423cb6c3fdb3d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/1b3eb626d5bb4d7684a813d35b9182ec, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/cff379d6cae9432986ac7f5ee68f5920, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/954f81bc0bab4067a8ddec7d322a2094, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1513811bad146e084ffff81dca3b4a0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/005f0e4d0feb444aa6b48707b163f750, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6da3eacb245241d78649fcdc3e560660, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/f7c800b5e99c4bba815a51cb555f0e08, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/7eff5d678d154fc6a152e1eae66473f0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/d1fc4fa75c86443bbc863c182ae446f8, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ff79772215474e7fb2954f516fcfdc7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/edb9a136806e482d888f32f90e3cbfda, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1f3203b6f664efaa069c161650292a3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/579a376e83ab4780b066cba48d7dfd58, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/5086c50391c0431798f2af091adbf772] to archive 2024-12-05T05:17:08,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:17:08,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/39aa317f5e894268927e92c8ebcf791a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/39aa317f5e894268927e92c8ebcf791a 2024-12-05T05:17:08,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/05f63486d53e4af98cbf94a777da4c7d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/05f63486d53e4af98cbf94a777da4c7d 2024-12-05T05:17:08,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b495cdada6344b7c8ebfe982cae48a2b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b495cdada6344b7c8ebfe982cae48a2b 2024-12-05T05:17:08,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ac4c684f63074edd82e2a11e75ee0ff0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ac4c684f63074edd82e2a11e75ee0ff0 2024-12-05T05:17:08,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/fd95b2d88603489e99ba0a07b128d4da to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/fd95b2d88603489e99ba0a07b128d4da 2024-12-05T05:17:08,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/55aa6cc29ec84401a08e2d2762b4e648 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/55aa6cc29ec84401a08e2d2762b4e648 2024-12-05T05:17:08,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/bb4e9704f20546ed8982d49070b48a54 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/bb4e9704f20546ed8982d49070b48a54 2024-12-05T05:17:08,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b70285c5bd9d43b9bf372d17a8fe6c2a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/b70285c5bd9d43b9bf372d17a8fe6c2a 2024-12-05T05:17:08,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/96fcb07dfb704e4f98e2828343b36590 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/96fcb07dfb704e4f98e2828343b36590 2024-12-05T05:17:08,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/668ddaaedb0f48d19a1a6627a64fa986 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/668ddaaedb0f48d19a1a6627a64fa986 2024-12-05T05:17:08,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6aa88a569187444bae0423cb6c3fdb3d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6aa88a569187444bae0423cb6c3fdb3d 2024-12-05T05:17:08,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/1b3eb626d5bb4d7684a813d35b9182ec to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/1b3eb626d5bb4d7684a813d35b9182ec 2024-12-05T05:17:08,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/cff379d6cae9432986ac7f5ee68f5920 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/cff379d6cae9432986ac7f5ee68f5920 2024-12-05T05:17:08,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/954f81bc0bab4067a8ddec7d322a2094 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/954f81bc0bab4067a8ddec7d322a2094 2024-12-05T05:17:08,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1513811bad146e084ffff81dca3b4a0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1513811bad146e084ffff81dca3b4a0 2024-12-05T05:17:08,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/005f0e4d0feb444aa6b48707b163f750 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/005f0e4d0feb444aa6b48707b163f750 2024-12-05T05:17:08,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6da3eacb245241d78649fcdc3e560660 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/6da3eacb245241d78649fcdc3e560660 2024-12-05T05:17:08,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/f7c800b5e99c4bba815a51cb555f0e08 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/f7c800b5e99c4bba815a51cb555f0e08 2024-12-05T05:17:08,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/7eff5d678d154fc6a152e1eae66473f0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/7eff5d678d154fc6a152e1eae66473f0 2024-12-05T05:17:08,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/d1fc4fa75c86443bbc863c182ae446f8 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/d1fc4fa75c86443bbc863c182ae446f8 2024-12-05T05:17:08,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ff79772215474e7fb2954f516fcfdc7d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/ff79772215474e7fb2954f516fcfdc7d 2024-12-05T05:17:08,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/16a7ee4b48ab4ff6ae7cdf03c2d7c978 2024-12-05T05:17:08,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/edb9a136806e482d888f32f90e3cbfda to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/edb9a136806e482d888f32f90e3cbfda 2024-12-05T05:17:08,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1f3203b6f664efaa069c161650292a3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/c1f3203b6f664efaa069c161650292a3 2024-12-05T05:17:08,337 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/579a376e83ab4780b066cba48d7dfd58 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/579a376e83ab4780b066cba48d7dfd58 2024-12-05T05:17:08,338 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/5086c50391c0431798f2af091adbf772 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/5086c50391c0431798f2af091adbf772 2024-12-05T05:17:08,339 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a45a95332aa246f6a5629527bf0e746a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/464f93fac7f54d91badc2f42613488f9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/aabceade6874459d9d1722f3d9fd844b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5670a0aaa8c14b74a6ce29e7c75e0ba0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2bae37dd525d42ebad26fc86a1d56ee8, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a244c9f390c94bb8b40ca1cee27f6279, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0425e36e1c5c4858bce70e538635c672, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f688e5ed55994e06970e4cbc8e55f565, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9c0c791e352c4e90a3ee1ecad5fa604f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/bec73c16b43d40c4a7fdc8b643d023c4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f6dfadfe3f9a43b9bfd717605bdc1269, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/18b9cc42b0484f87b01c3bd9119e0ce3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/7224fc50cbb643c58138b7d702baa5e3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/b3274e508c444893ac107833c9b34453, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/cbf1476667f749f2b0f05f01d53cb101, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/6baa642623804686bbe7c4b1e5b6c96f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2d23b4600da94ee0b1c267b8e2b979e0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/e70b8d74a8ab483d9fb595beffd23e49, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/272d19dd07824d60966b21dfaf27f3a4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/26d110b874704dfabc9a55c807d24a75, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/451980f04aff4964a22e09e44e356c9e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2f29633b3a5c4f52905cdc7f19ae5d0b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/d093a984b0b74111a969b8aea17254ec, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5f5a3c4a9fc84a7f8fdce1da4871604b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9e437f46ad5145f6b267450a079b923d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/67829e132a9b4cd78ab3a2da212bc569] to archive 2024-12-05T05:17:08,340 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:17:08,342 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a45a95332aa246f6a5629527bf0e746a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a45a95332aa246f6a5629527bf0e746a 2024-12-05T05:17:08,344 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/464f93fac7f54d91badc2f42613488f9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/464f93fac7f54d91badc2f42613488f9 2024-12-05T05:17:08,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/aabceade6874459d9d1722f3d9fd844b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/aabceade6874459d9d1722f3d9fd844b 2024-12-05T05:17:08,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5670a0aaa8c14b74a6ce29e7c75e0ba0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5670a0aaa8c14b74a6ce29e7c75e0ba0 2024-12-05T05:17:08,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2bae37dd525d42ebad26fc86a1d56ee8 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2bae37dd525d42ebad26fc86a1d56ee8 2024-12-05T05:17:08,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a244c9f390c94bb8b40ca1cee27f6279 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/a244c9f390c94bb8b40ca1cee27f6279 2024-12-05T05:17:08,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0425e36e1c5c4858bce70e538635c672 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0425e36e1c5c4858bce70e538635c672 2024-12-05T05:17:08,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f688e5ed55994e06970e4cbc8e55f565 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f688e5ed55994e06970e4cbc8e55f565 2024-12-05T05:17:08,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9c0c791e352c4e90a3ee1ecad5fa604f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9c0c791e352c4e90a3ee1ecad5fa604f 2024-12-05T05:17:08,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/bec73c16b43d40c4a7fdc8b643d023c4 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/bec73c16b43d40c4a7fdc8b643d023c4 2024-12-05T05:17:08,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f6dfadfe3f9a43b9bfd717605bdc1269 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/f6dfadfe3f9a43b9bfd717605bdc1269 2024-12-05T05:17:08,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/18b9cc42b0484f87b01c3bd9119e0ce3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/18b9cc42b0484f87b01c3bd9119e0ce3 2024-12-05T05:17:08,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/7224fc50cbb643c58138b7d702baa5e3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/7224fc50cbb643c58138b7d702baa5e3 2024-12-05T05:17:08,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/b3274e508c444893ac107833c9b34453 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/b3274e508c444893ac107833c9b34453 2024-12-05T05:17:08,363 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/cbf1476667f749f2b0f05f01d53cb101 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/cbf1476667f749f2b0f05f01d53cb101 2024-12-05T05:17:08,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/6baa642623804686bbe7c4b1e5b6c96f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/6baa642623804686bbe7c4b1e5b6c96f 2024-12-05T05:17:08,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2d23b4600da94ee0b1c267b8e2b979e0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2d23b4600da94ee0b1c267b8e2b979e0 2024-12-05T05:17:08,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/e70b8d74a8ab483d9fb595beffd23e49 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/e70b8d74a8ab483d9fb595beffd23e49 2024-12-05T05:17:08,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/272d19dd07824d60966b21dfaf27f3a4 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/272d19dd07824d60966b21dfaf27f3a4 2024-12-05T05:17:08,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/26d110b874704dfabc9a55c807d24a75 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/26d110b874704dfabc9a55c807d24a75 2024-12-05T05:17:08,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/451980f04aff4964a22e09e44e356c9e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/451980f04aff4964a22e09e44e356c9e 2024-12-05T05:17:08,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2f29633b3a5c4f52905cdc7f19ae5d0b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/2f29633b3a5c4f52905cdc7f19ae5d0b 2024-12-05T05:17:08,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/d093a984b0b74111a969b8aea17254ec to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/d093a984b0b74111a969b8aea17254ec 2024-12-05T05:17:08,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5f5a3c4a9fc84a7f8fdce1da4871604b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/5f5a3c4a9fc84a7f8fdce1da4871604b 2024-12-05T05:17:08,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9e437f46ad5145f6b267450a079b923d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/9e437f46ad5145f6b267450a079b923d 2024-12-05T05:17:08,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/67829e132a9b4cd78ab3a2da212bc569 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/67829e132a9b4cd78ab3a2da212bc569 2024-12-05T05:17:08,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/567090afe8514cd5b2486eac8b6e792d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ead958cc1d0a45fe95132e1b8b84f9ff, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/91196877312446feadad3d743e3775ee, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/517a33c0a13f4c908696cbba3fe91d97, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/2a6a65ad0ee94e0b92c8830f3b3f4762, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e538550b95864eaab453e7e346f75273, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/dded4e4c8aef4a2a8429f0797251da06, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/68cfca1edd4b4fd1a69a57f6c8b410af, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/23bb963814c349448bf237467e9ff5e2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e053d8d741534db991a1ee0de6ac59a0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/7e79af0a83ed43cba0d8aec59e6cafdd, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/df44226e9f314a5e9ca004e48940445c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1b79cdcef2c04738a05f9cc6cb3d034a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/bbe3b02063284191a5097d9cfad34d2d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/51f8551757534c3bb6e95b2b4648a2a9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/61b97f69722249819d6ab63327c942bb, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/99538fc4a18e4607839779995362eb76, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/452356bdaeaa49988a35a82f2a572692, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/6939bca485a64e77a1a5c0d229912a29, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/b1b38ba20d984aa6b54b58608e91e550, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/90d3b311ee53441cad63a88d924cf29d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1868824a44af4364bf1a62336437e663, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/a6baec13706e462da690177179bd14da, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ca1902d54cf2460b9763672430fcf5b6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/8ff7847586e5438d9f879dda52b5e1d5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/de483746d63c47c8852bd07b64b40fa9] to archive 2024-12-05T05:17:08,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:17:08,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/567090afe8514cd5b2486eac8b6e792d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/567090afe8514cd5b2486eac8b6e792d 2024-12-05T05:17:08,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ead958cc1d0a45fe95132e1b8b84f9ff to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ead958cc1d0a45fe95132e1b8b84f9ff 2024-12-05T05:17:08,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/91196877312446feadad3d743e3775ee to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/91196877312446feadad3d743e3775ee 2024-12-05T05:17:08,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/517a33c0a13f4c908696cbba3fe91d97 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/517a33c0a13f4c908696cbba3fe91d97 2024-12-05T05:17:08,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/2a6a65ad0ee94e0b92c8830f3b3f4762 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/2a6a65ad0ee94e0b92c8830f3b3f4762 2024-12-05T05:17:08,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e538550b95864eaab453e7e346f75273 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e538550b95864eaab453e7e346f75273 2024-12-05T05:17:08,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/dded4e4c8aef4a2a8429f0797251da06 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/dded4e4c8aef4a2a8429f0797251da06 2024-12-05T05:17:08,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/68cfca1edd4b4fd1a69a57f6c8b410af to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/68cfca1edd4b4fd1a69a57f6c8b410af 2024-12-05T05:17:08,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/23bb963814c349448bf237467e9ff5e2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/23bb963814c349448bf237467e9ff5e2 2024-12-05T05:17:08,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e053d8d741534db991a1ee0de6ac59a0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/e053d8d741534db991a1ee0de6ac59a0 2024-12-05T05:17:08,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/7e79af0a83ed43cba0d8aec59e6cafdd to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/7e79af0a83ed43cba0d8aec59e6cafdd 2024-12-05T05:17:08,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/df44226e9f314a5e9ca004e48940445c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/df44226e9f314a5e9ca004e48940445c 2024-12-05T05:17:08,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1b79cdcef2c04738a05f9cc6cb3d034a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1b79cdcef2c04738a05f9cc6cb3d034a 2024-12-05T05:17:08,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/bbe3b02063284191a5097d9cfad34d2d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/bbe3b02063284191a5097d9cfad34d2d 2024-12-05T05:17:08,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/51f8551757534c3bb6e95b2b4648a2a9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/51f8551757534c3bb6e95b2b4648a2a9 2024-12-05T05:17:08,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/61b97f69722249819d6ab63327c942bb to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/61b97f69722249819d6ab63327c942bb 2024-12-05T05:17:08,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/99538fc4a18e4607839779995362eb76 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/99538fc4a18e4607839779995362eb76 2024-12-05T05:17:08,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/452356bdaeaa49988a35a82f2a572692 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/452356bdaeaa49988a35a82f2a572692 2024-12-05T05:17:08,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/6939bca485a64e77a1a5c0d229912a29 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/6939bca485a64e77a1a5c0d229912a29 2024-12-05T05:17:08,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/b1b38ba20d984aa6b54b58608e91e550 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/b1b38ba20d984aa6b54b58608e91e550 2024-12-05T05:17:08,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/90d3b311ee53441cad63a88d924cf29d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/90d3b311ee53441cad63a88d924cf29d 2024-12-05T05:17:08,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1868824a44af4364bf1a62336437e663 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/1868824a44af4364bf1a62336437e663 2024-12-05T05:17:08,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/a6baec13706e462da690177179bd14da to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/a6baec13706e462da690177179bd14da 2024-12-05T05:17:08,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ca1902d54cf2460b9763672430fcf5b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/ca1902d54cf2460b9763672430fcf5b6 2024-12-05T05:17:08,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/8ff7847586e5438d9f879dda52b5e1d5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/8ff7847586e5438d9f879dda52b5e1d5 2024-12-05T05:17:08,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/de483746d63c47c8852bd07b64b40fa9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/de483746d63c47c8852bd07b64b40fa9 2024-12-05T05:17:08,406 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/recovered.edits/474.seqid, newMaxSeqId=474, maxSeqId=4 2024-12-05T05:17:08,407 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. 2024-12-05T05:17:08,407 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] regionserver.HRegion(1676): Region close journal for 09d09c462e929eda7f34ea9ccdd56981: Waiting for close lock at 1733375826226Running coprocessor pre-close hooks at 1733375826226Disabling compacts and flushes for region at 1733375826226Disabling writes for close at 1733375826644 (+418 ms)Obtaining lock to block concurrent updates at 1733375826644Preparing flush snapshotting stores in 09d09c462e929eda7f34ea9ccdd56981 at 1733375826644Finished memstore snapshotting TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733375826644Flushing stores of TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981. at 1733375826644Flushing 09d09c462e929eda7f34ea9ccdd56981/A: creating writer at 1733375826644Flushing mob file 09d09c462e929eda7f34ea9ccdd56981/A: appending metadata at 1733375826649 (+5 ms)Flushing mob file 09d09c462e929eda7f34ea9ccdd56981/A: closing flushed file at 1733375826649Flushing 09d09c462e929eda7f34ea9ccdd56981/A: appending metadata at 1733375827057 (+408 ms)Flushing 09d09c462e929eda7f34ea9ccdd56981/A: closing flushed file at 1733375827057Flushing 09d09c462e929eda7f34ea9ccdd56981/B: creating writer at 1733375827466 (+409 ms)Flushing 09d09c462e929eda7f34ea9ccdd56981/B: appending metadata at 1733375827469 (+3 ms)Flushing 09d09c462e929eda7f34ea9ccdd56981/B: closing flushed file at 1733375827469Flushing 09d09c462e929eda7f34ea9ccdd56981/C: creating writer at 1733375827878 (+409 ms)Flushing 09d09c462e929eda7f34ea9ccdd56981/C: appending metadata at 1733375827881 (+3 ms)Flushing 09d09c462e929eda7f34ea9ccdd56981/C: closing flushed file at 1733375827881Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a3cd417: reopening flushed file at 1733375828289 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64dfda58: reopening flushed file at 1733375828296 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@375fc836: reopening flushed file at 1733375828302 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 09d09c462e929eda7f34ea9ccdd56981 in 1663ms, sequenceid=471, compaction requested=true at 1733375828307 (+5 ms)Writing region close event to WAL at 1733375828402 (+95 ms)Running coprocessor post-close hooks at 1733375828407 (+5 ms)Closed at 1733375828407 2024-12-05T05:17:08,410 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=53}] handler.UnassignRegionHandler(157): Closed 09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:08,410 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=09d09c462e929eda7f34ea9ccdd56981, regionState=CLOSED 2024-12-05T05:17:08,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:17:08,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-05T05:17:08,413 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseRegionProcedure 09d09c462e929eda7f34ea9ccdd56981, server=8edcaf2ecc84,36233,1733375771401 in 2.3380 sec 2024-12-05T05:17:08,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-12-05T05:17:08,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09d09c462e929eda7f34ea9ccdd56981, UNASSIGN in 2.3420 sec 2024-12-05T05:17:08,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-05T05:17:08,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3450 sec 2024-12-05T05:17:08,418 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375828417"}]},"ts":"1733375828417"} 2024-12-05T05:17:08,419 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T05:17:08,419 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T05:17:08,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3550 sec 2024-12-05T05:17:09,517 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T05:17:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-05T05:17:10,224 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:10,225 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T05:17:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:10,227 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=54, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:10,229 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=54, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T05:17:10,236 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,239 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/recovered.edits] 2024-12-05T05:17:10,244 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/0d6c0b6c1950439a8beec2e1ba3e4bbe to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/0d6c0b6c1950439a8beec2e1ba3e4bbe 2024-12-05T05:17:10,246 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/587081a430d346f69bb1013e72e79841 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/587081a430d346f69bb1013e72e79841 2024-12-05T05:17:10,248 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/dc0c112fccf445fda1634548dda1d88e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/A/dc0c112fccf445fda1634548dda1d88e 2024-12-05T05:17:10,252 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0496ffea49874ec48c62638c85fc1b04 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/0496ffea49874ec48c62638c85fc1b04 2024-12-05T05:17:10,254 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/24bd27062dba45efb3a958d9126eb3cb to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/24bd27062dba45efb3a958d9126eb3cb 2024-12-05T05:17:10,256 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/621f5d222d164e8bafb826b81cf24c54 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/B/621f5d222d164e8bafb826b81cf24c54 2024-12-05T05:17:10,261 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/4c9dd798a87a44f5a474524e8decec24 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/4c9dd798a87a44f5a474524e8decec24 2024-12-05T05:17:10,265 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/9232dec37f9e4850ad36036d533317f5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/9232dec37f9e4850ad36036d533317f5 2024-12-05T05:17:10,268 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/cce33f9d848743eb89e594fca9262719 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/C/cce33f9d848743eb89e594fca9262719 2024-12-05T05:17:10,273 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/recovered.edits/474.seqid to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981/recovered.edits/474.seqid 2024-12-05T05:17:10,275 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,276 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T05:17:10,276 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T05:17:10,278 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T05:17:10,289 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205040c563976ac4c388ca6934554d18bd6_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205040c563976ac4c388ca6934554d18bd6_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,291 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120507e5d3471d424677ad5952fa1b024b6c_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120507e5d3471d424677ad5952fa1b024b6c_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,294 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ad13ca8036c4f4baea524d2002ca8e7_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ad13ca8036c4f4baea524d2002ca8e7_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,297 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052aff7cfe73a34018921832d2d6ff54a7_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052aff7cfe73a34018921832d2d6ff54a7_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,299 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ec22024c51c415cb8320d2f659e9003_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412052ec22024c51c415cb8320d2f659e9003_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,303 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120535799a99ab9f4cbeb7cc5d1a37b0d25e_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120535799a99ab9f4cbeb7cc5d1a37b0d25e_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a59ff94915949ec97bc0c4147826177_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053a59ff94915949ec97bc0c4147826177_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,308 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054618eecb7fc742248aa1b225b29682a9_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054618eecb7fc742248aa1b225b29682a9_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,310 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054df908453c87453aa7cb5f63ed84b3ab_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054df908453c87453aa7cb5f63ed84b3ab_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,311 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205530286856fb4449db1f324a2d2318d40_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205530286856fb4449db1f324a2d2318d40_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,312 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205643b588c1d49454a9e075fa92c245495_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205643b588c1d49454a9e075fa92c245495_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,313 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205685d3ce2780644eba6268063b29fd985_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205685d3ce2780644eba6268063b29fd985_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,315 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120585fba102a4db4a858b02e9fc54b914a0_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120585fba102a4db4a858b02e9fc54b914a0_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,318 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205903109d802c84922ba4e6af858419274_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205903109d802c84922ba4e6af858419274_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,320 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120592da0344cf4846f081a0c4b01c216b84_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120592da0344cf4846f081a0c4b01c216b84_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,322 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205be46ec04ee8a45708cc80ff37af749c8_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205be46ec04ee8a45708cc80ff37af749c8_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,323 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d1de19b2e52a498da712ecc6987dd3b6_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205d1de19b2e52a498da712ecc6987dd3b6_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,328 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f10c861eeb174a33a85d600b5eb8a147_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f10c861eeb174a33a85d600b5eb8a147_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,330 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f1e5e1de38074cb19ab6e462afdc0f70_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f1e5e1de38074cb19ab6e462afdc0f70_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,332 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f3217286b6c14071930ac5be50d28c48_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f3217286b6c14071930ac5be50d28c48_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T05:17:10,335 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff570749aab54191baf6025a84820ba7_09d09c462e929eda7f34ea9ccdd56981 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ff570749aab54191baf6025a84820ba7_09d09c462e929eda7f34ea9ccdd56981 2024-12-05T05:17:10,336 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T05:17:10,348 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=54, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:10,353 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T05:17:10,356 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T05:17:10,358 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=54, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:10,358 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T05:17:10,358 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733375830358"}]},"ts":"9223372036854775807"} 2024-12-05T05:17:10,361 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T05:17:10,361 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 09d09c462e929eda7f34ea9ccdd56981, NAME => 'TestAcidGuarantees,,1733375801690.09d09c462e929eda7f34ea9ccdd56981.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T05:17:10,361 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T05:17:10,362 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733375830361"}]},"ts":"9223372036854775807"} 2024-12-05T05:17:10,365 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T05:17:10,368 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=54, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:10,370 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 143 msec 2024-12-05T05:17:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-05T05:17:10,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T05:17:10,545 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:10,571 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250 (was 249) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_195128769_22 at /127.0.0.1:48628 [Waiting for operation #575] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2050451830_22 at /127.0.0.1:52250 [Waiting for operation #276] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=794 (was 846), ProcessCount=11 (was 11), AvailableMemoryMB=3676 (was 3118) - AvailableMemoryMB LEAK? - 2024-12-05T05:17:10,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=794, ProcessCount=11, AvailableMemoryMB=3673 2024-12-05T05:17:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:17:10,603 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:17:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:10,611 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T05:17:10,611 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:10,613 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T05:17:10,613 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 55 2024-12-05T05:17:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-05T05:17:10,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742061_1237 (size=1061) 2024-12-05T05:17:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-05T05:17:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-05T05:17:10,939 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:17:11,076 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 2024-12-05T05:17:11,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742062_1238 (size=53) 2024-12-05T05:17:11,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-05T05:17:11,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:17:11,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ac783a93e1fe6938afb704b21f8b8675, disabling compactions & flushes 2024-12-05T05:17:11,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:11,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:11,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. after waiting 0 ms 2024-12-05T05:17:11,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:11,511 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:11,511 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ac783a93e1fe6938afb704b21f8b8675: Waiting for close lock at 1733375831510Disabling compacts and flushes for region at 1733375831510Disabling writes for close at 1733375831511 (+1 ms)Writing region close event to WAL at 1733375831511Closed at 1733375831511 2024-12-05T05:17:11,512 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T05:17:11,512 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733375831512"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733375831512"}]},"ts":"1733375831512"} 2024-12-05T05:17:11,514 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T05:17:11,515 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T05:17:11,515 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375831515"}]},"ts":"1733375831515"} 2024-12-05T05:17:11,517 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T05:17:11,518 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, ASSIGN}] 2024-12-05T05:17:11,519 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, ASSIGN 2024-12-05T05:17:11,520 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, ASSIGN; state=OFFLINE, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=false 2024-12-05T05:17:11,671 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=ac783a93e1fe6938afb704b21f8b8675, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:11,673 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, ASSIGN because future has completed 2024-12-05T05:17:11,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:17:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-05T05:17:11,833 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:11,834 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7752): Opening region: {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:17:11,834 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,834 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:17:11,834 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7794): checking encryption for ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,835 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(7797): checking classloading for ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,840 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,842 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:11,842 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac783a93e1fe6938afb704b21f8b8675 columnFamilyName A 2024-12-05T05:17:11,842 DEBUG [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:11,851 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.HStore(327): Store=ac783a93e1fe6938afb704b21f8b8675/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:11,852 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,856 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:11,856 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac783a93e1fe6938afb704b21f8b8675 columnFamilyName B 2024-12-05T05:17:11,856 DEBUG [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:11,857 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.HStore(327): Store=ac783a93e1fe6938afb704b21f8b8675/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:11,857 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,859 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:11,860 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac783a93e1fe6938afb704b21f8b8675 columnFamilyName C 2024-12-05T05:17:11,860 DEBUG [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:11,864 INFO [StoreOpener-ac783a93e1fe6938afb704b21f8b8675-1 {}] regionserver.HStore(327): Store=ac783a93e1fe6938afb704b21f8b8675/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:11,865 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:11,865 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1038): replaying wal for ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,866 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,867 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,868 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1048): stopping wal replay for ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,868 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1060): Cleaning up temporary data for ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,869 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:17:11,870 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1093): writing seq id for ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,892 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T05:17:11,893 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1114): Opened ac783a93e1fe6938afb704b21f8b8675; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67843039, jitterRate=0.010940060019493103}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:17:11,893 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:11,894 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegion(1006): Region open journal for ac783a93e1fe6938afb704b21f8b8675: Running coprocessor pre-open hook at 1733375831835Writing region info on filesystem at 1733375831835Initializing all the Stores at 1733375831839 (+4 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375831839Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375831839Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375831839Cleaning up temporary data from old regions at 1733375831868 (+29 ms)Running coprocessor post-open hooks at 1733375831893 (+25 ms)Region opened successfully at 1733375831894 (+1 ms) 2024-12-05T05:17:11,895 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., pid=57, masterSystemTime=1733375831828 2024-12-05T05:17:11,897 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:11,897 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=57}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:11,897 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=ac783a93e1fe6938afb704b21f8b8675, regionState=OPEN, openSeqNum=2, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:11,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:17:11,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-05T05:17:11,901 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; OpenRegionProcedure ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 in 227 msec 2024-12-05T05:17:11,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-12-05T05:17:11,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, ASSIGN in 383 msec 2024-12-05T05:17:11,904 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T05:17:11,905 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375831904"}]},"ts":"1733375831904"} 2024-12-05T05:17:11,907 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T05:17:11,908 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=55, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T05:17:11,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3040 sec 2024-12-05T05:17:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-05T05:17:12,763 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:12,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d622426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a66ac1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,767 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4977f21f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,768 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,770 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,771 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T05:17:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 8edcaf2ecc84,46695,1733375770505 2024-12-05T05:17:12,771 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@216fa451 2024-12-05T05:17:12,772 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T05:17:12,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T05:17:12,774 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e0d1e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,774 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,774 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,775 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8b5bdf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,776 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,776 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79e20013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ba03b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,778 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,778 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,779 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241dfc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,780 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,780 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c8ccfcd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,782 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3814b668, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,782 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,782 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,783 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e308374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,784 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,785 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f4eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,786 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4c8c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,786 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,786 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,787 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec5f777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,788 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54944, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c51bf23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c35cb78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,791 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ed0076a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,793 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53732237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a9105f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,795 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f4f8614, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,796 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a21c040, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7d7961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,798 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a74c12b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,800 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26082195, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d97a88a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,802 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,802 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,803 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5002b38b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,805 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11e59b8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,806 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e309a51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,806 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,806 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,807 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32662dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,807 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,808 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8cebb1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8cf2b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:12,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:12,811 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11693811, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:12,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:12,812 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:12,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1656e62e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:12,813 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,813 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,813 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,813 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,813 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,814 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,815 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,817 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,817 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,817 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:12,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,817 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,818 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:12,821 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:12,821 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-12-05T05:17:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T05:17:12,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,830 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:12,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:17:12,832 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:12,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:12,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:17:12,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:12,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:12,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:12,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:12,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:12,844 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:12,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/5ac76b7c712f4108b1b9d7f15ebbae78 is 50, key is test_row_0/A:col10/1733375832835/Put/seqid=0 2024-12-05T05:17:12,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742063_1239 (size=12001) 2024-12-05T05:17:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:12,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375892918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:12,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375892922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:12,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375892923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375892924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:12,928 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:12,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375892924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:12,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:12,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T05:17:12,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T05:17:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:12,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:12,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375893034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375893035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375893035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375893035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375893035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T05:17:13,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T05:17:13,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:13,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375893243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375893243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375893243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375893244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375893244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/5ac76b7c712f4108b1b9d7f15ebbae78 2024-12-05T05:17:13,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T05:17:13,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:13,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/63b5f150a1e5467693cfe5cd5306c5b1 is 50, key is test_row_0/B:col10/1733375832835/Put/seqid=0 2024-12-05T05:17:13,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742064_1240 (size=12001) 2024-12-05T05:17:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T05:17:13,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T05:17:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375893553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375893553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375893553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375893553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375893553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,556 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:13,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T05:17:13,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/63b5f150a1e5467693cfe5cd5306c5b1 2024-12-05T05:17:13,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T05:17:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:13,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/f4a51583b6524a31a8a96ba446f0895e is 50, key is test_row_0/C:col10/1733375832835/Put/seqid=0 2024-12-05T05:17:13,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742065_1241 (size=12001) 2024-12-05T05:17:13,828 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/f4a51583b6524a31a8a96ba446f0895e 2024-12-05T05:17:13,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/5ac76b7c712f4108b1b9d7f15ebbae78 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5ac76b7c712f4108b1b9d7f15ebbae78 2024-12-05T05:17:13,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5ac76b7c712f4108b1b9d7f15ebbae78, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T05:17:13,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/63b5f150a1e5467693cfe5cd5306c5b1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/63b5f150a1e5467693cfe5cd5306c5b1 2024-12-05T05:17:13,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/63b5f150a1e5467693cfe5cd5306c5b1, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T05:17:13,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/f4a51583b6524a31a8a96ba446f0895e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f4a51583b6524a31a8a96ba446f0895e 2024-12-05T05:17:13,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f4a51583b6524a31a8a96ba446f0895e, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T05:17:13,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ac783a93e1fe6938afb704b21f8b8675 in 1066ms, sequenceid=14, compaction requested=false 2024-12-05T05:17:13,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:13,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-12-05T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:13,920 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:13,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:13,927 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T05:17:13,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/6497a1b420e04f9ea068dd9bfe21a35c is 50, key is test_row_0/A:col10/1733375832921/Put/seqid=0 2024-12-05T05:17:13,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T05:17:13,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742066_1242 (size=12001) 2024-12-05T05:17:13,990 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/6497a1b420e04f9ea068dd9bfe21a35c 2024-12-05T05:17:14,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/66d2c8f726b54296a0f45f516dddc80c is 50, key is test_row_0/B:col10/1733375832921/Put/seqid=0 2024-12-05T05:17:14,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742067_1243 (size=12001) 2024-12-05T05:17:14,068 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/66d2c8f726b54296a0f45f516dddc80c 2024-12-05T05:17:14,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:14,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/7270eb9f183441f89739d422610cd53c is 50, key is test_row_0/C:col10/1733375832921/Put/seqid=0 2024-12-05T05:17:14,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742068_1244 (size=12001) 2024-12-05T05:17:14,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375894111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375894121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375894134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375894134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375894134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,148 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/7270eb9f183441f89739d422610cd53c 2024-12-05T05:17:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/6497a1b420e04f9ea068dd9bfe21a35c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6497a1b420e04f9ea068dd9bfe21a35c 2024-12-05T05:17:14,161 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6497a1b420e04f9ea068dd9bfe21a35c, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T05:17:14,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/66d2c8f726b54296a0f45f516dddc80c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/66d2c8f726b54296a0f45f516dddc80c 2024-12-05T05:17:14,170 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/66d2c8f726b54296a0f45f516dddc80c, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T05:17:14,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/7270eb9f183441f89739d422610cd53c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/7270eb9f183441f89739d422610cd53c 2024-12-05T05:17:14,200 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/7270eb9f183441f89739d422610cd53c, entries=150, sequenceid=33, filesize=11.7 K 2024-12-05T05:17:14,202 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=80.51 KB/82440 for ac783a93e1fe6938afb704b21f8b8675 in 282ms, sequenceid=33, compaction requested=false 2024-12-05T05:17:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:14,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-12-05T05:17:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-12-05T05:17:14,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-05T05:17:14,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3720 sec 2024-12-05T05:17:14,216 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 1.3990 sec 2024-12-05T05:17:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:14,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T05:17:14,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:14,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:14,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:14,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:14,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:14,252 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:14,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/135803f8dcef470485b0adfb51ea88c6 is 50, key is test_row_0/A:col10/1733375834130/Put/seqid=0 2024-12-05T05:17:14,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375894288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375894291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375894291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,303 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375894294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,304 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742069_1245 (size=16681) 2024-12-05T05:17:14,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/135803f8dcef470485b0adfb51ea88c6 2024-12-05T05:17:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,311 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375894311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/06d4c9c77ab04472bbeb450cb2744e7d is 50, key is test_row_0/B:col10/1733375834130/Put/seqid=0 2024-12-05T05:17:14,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742070_1246 (size=12001) 2024-12-05T05:17:14,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/06d4c9c77ab04472bbeb450cb2744e7d 2024-12-05T05:17:14,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ac6a63dc3add4c828224a41a9cf9cf89 is 50, key is test_row_0/C:col10/1733375834130/Put/seqid=0 2024-12-05T05:17:14,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742071_1247 (size=12001) 2024-12-05T05:17:14,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ac6a63dc3add4c828224a41a9cf9cf89 2024-12-05T05:17:14,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/135803f8dcef470485b0adfb51ea88c6 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/135803f8dcef470485b0adfb51ea88c6 2024-12-05T05:17:14,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375894412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375894413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/135803f8dcef470485b0adfb51ea88c6, entries=250, sequenceid=50, filesize=16.3 K 2024-12-05T05:17:14,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/06d4c9c77ab04472bbeb450cb2744e7d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/06d4c9c77ab04472bbeb450cb2744e7d 2024-12-05T05:17:14,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375894415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375894421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/06d4c9c77ab04472bbeb450cb2744e7d, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T05:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375894423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ac6a63dc3add4c828224a41a9cf9cf89 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ac6a63dc3add4c828224a41a9cf9cf89 2024-12-05T05:17:14,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ac6a63dc3add4c828224a41a9cf9cf89, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T05:17:14,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for ac783a93e1fe6938afb704b21f8b8675 in 192ms, sequenceid=50, compaction requested=true 2024-12-05T05:17:14,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:14,443 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:14,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:14,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:14,444 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:14,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:14,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:14,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:14,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:14,448 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:14,449 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:14,449 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:14,450 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5ac76b7c712f4108b1b9d7f15ebbae78, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6497a1b420e04f9ea068dd9bfe21a35c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/135803f8dcef470485b0adfb51ea88c6] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=39.7 K 2024-12-05T05:17:14,452 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:14,452 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:14,452 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:14,452 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/63b5f150a1e5467693cfe5cd5306c5b1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/66d2c8f726b54296a0f45f516dddc80c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/06d4c9c77ab04472bbeb450cb2744e7d] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=35.2 K 2024-12-05T05:17:14,453 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ac76b7c712f4108b1b9d7f15ebbae78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375832835 2024-12-05T05:17:14,453 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6497a1b420e04f9ea068dd9bfe21a35c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733375832920 2024-12-05T05:17:14,453 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 63b5f150a1e5467693cfe5cd5306c5b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375832835 2024-12-05T05:17:14,454 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 135803f8dcef470485b0adfb51ea88c6, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375834108 2024-12-05T05:17:14,454 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d2c8f726b54296a0f45f516dddc80c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733375832920 2024-12-05T05:17:14,454 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 06d4c9c77ab04472bbeb450cb2744e7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375834130 2024-12-05T05:17:14,471 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:14,472 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/313136bebb3a4b37832e8b9cc8a7e9e1 is 50, key is test_row_0/A:col10/1733375834130/Put/seqid=0 2024-12-05T05:17:14,491 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:14,491 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/70265c04bce544dc8bfdfcf1788840ca is 50, key is test_row_0/B:col10/1733375834130/Put/seqid=0 2024-12-05T05:17:14,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742072_1248 (size=12104) 2024-12-05T05:17:14,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742073_1249 (size=12104) 2024-12-05T05:17:14,541 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/70265c04bce544dc8bfdfcf1788840ca as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/70265c04bce544dc8bfdfcf1788840ca 2024-12-05T05:17:14,542 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/313136bebb3a4b37832e8b9cc8a7e9e1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/313136bebb3a4b37832e8b9cc8a7e9e1 2024-12-05T05:17:14,577 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 70265c04bce544dc8bfdfcf1788840ca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:14,577 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:14,577 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=13, startTime=1733375834444; duration=0sec 2024-12-05T05:17:14,577 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:14,577 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:14,578 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:14,578 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into 313136bebb3a4b37832e8b9cc8a7e9e1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:14,579 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:14,580 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:14,580 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:14,580 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f4a51583b6524a31a8a96ba446f0895e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/7270eb9f183441f89739d422610cd53c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ac6a63dc3add4c828224a41a9cf9cf89] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=35.2 K 2024-12-05T05:17:14,580 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f4a51583b6524a31a8a96ba446f0895e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375832835 2024-12-05T05:17:14,581 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7270eb9f183441f89739d422610cd53c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1733375832920 2024-12-05T05:17:14,581 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:14,581 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=13, startTime=1733375834443; duration=0sec 2024-12-05T05:17:14,581 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:14,581 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:14,581 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6a63dc3add4c828224a41a9cf9cf89, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375834130 2024-12-05T05:17:14,618 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:14,619 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/42621997bed64497a22994f79c093dd0 is 50, key is test_row_0/C:col10/1733375834130/Put/seqid=0 2024-12-05T05:17:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:14,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T05:17:14,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:14,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:14,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:14,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:14,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:14,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:14,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/fbe6715473894acd88da9865ee84ae40 is 50, key is test_row_0/A:col10/1733375834288/Put/seqid=0 2024-12-05T05:17:14,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742074_1250 (size=12104) 2024-12-05T05:17:14,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375894654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375894655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375894656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375894658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375894665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,672 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742075_1251 (size=12001) 2024-12-05T05:17:14,683 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/42621997bed64497a22994f79c093dd0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/42621997bed64497a22994f79c093dd0 2024-12-05T05:17:14,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/fbe6715473894acd88da9865ee84ae40 2024-12-05T05:17:14,692 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into 42621997bed64497a22994f79c093dd0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:14,692 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:14,692 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=13, startTime=1733375834445; duration=0sec 2024-12-05T05:17:14,692 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:14,692 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:14,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/8e0c2e93e62b480f8481b1a556414098 is 50, key is test_row_0/B:col10/1733375834288/Put/seqid=0 2024-12-05T05:17:14,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742076_1252 (size=12001) 2024-12-05T05:17:14,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/8e0c2e93e62b480f8481b1a556414098 2024-12-05T05:17:14,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/d86a9ecd60a645f2a7b9e8f55fae5546 is 50, key is test_row_0/C:col10/1733375834288/Put/seqid=0 2024-12-05T05:17:14,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375894774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375894783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742077_1253 (size=12001) 2024-12-05T05:17:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-05T05:17:14,973 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:14,973 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-12-05T05:17:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T05:17:14,978 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:14,979 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:14,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:14,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375894984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375894984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375894984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,985 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375894988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:14,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375894994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:14,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T05:17:15,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T05:17:15,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:15,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:15,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:15,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:15,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:15,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/d86a9ecd60a645f2a7b9e8f55fae5546 2024-12-05T05:17:15,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/fbe6715473894acd88da9865ee84ae40 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/fbe6715473894acd88da9865ee84ae40 2024-12-05T05:17:15,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/fbe6715473894acd88da9865ee84ae40, entries=150, sequenceid=68, filesize=11.7 K 2024-12-05T05:17:15,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/8e0c2e93e62b480f8481b1a556414098 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8e0c2e93e62b480f8481b1a556414098 2024-12-05T05:17:15,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8e0c2e93e62b480f8481b1a556414098, entries=150, sequenceid=68, filesize=11.7 K 2024-12-05T05:17:15,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/d86a9ecd60a645f2a7b9e8f55fae5546 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d86a9ecd60a645f2a7b9e8f55fae5546 2024-12-05T05:17:15,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d86a9ecd60a645f2a7b9e8f55fae5546, entries=150, sequenceid=68, filesize=11.7 K 2024-12-05T05:17:15,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ac783a93e1fe6938afb704b21f8b8675 in 614ms, sequenceid=68, compaction requested=false 2024-12-05T05:17:15,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:15,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-12-05T05:17:15,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:15,288 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-05T05:17:15,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:15,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:15,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:15,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:15,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:15,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:15,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/6b6ad38f6fa24dc2908aa13893676564 is 50, key is test_row_0/A:col10/1733375834653/Put/seqid=0 2024-12-05T05:17:15,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T05:17:15,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:15,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:15,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375895324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375895325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,333 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375895327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375895325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,335 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742078_1254 (size=12001) 2024-12-05T05:17:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375895332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375895432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375895443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375895443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375895443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375895453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T05:17:15,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375895653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375895653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375895653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375895654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375895663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,738 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/6b6ad38f6fa24dc2908aa13893676564 2024-12-05T05:17:15,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c53c0b7cb855419dbeec9bec408659ba is 50, key is test_row_0/B:col10/1733375834653/Put/seqid=0 2024-12-05T05:17:15,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742079_1255 (size=12001) 2024-12-05T05:17:15,800 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c53c0b7cb855419dbeec9bec408659ba 2024-12-05T05:17:15,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e31e47e8efea4a6b9d218ea35c25924d is 50, key is test_row_0/C:col10/1733375834653/Put/seqid=0 2024-12-05T05:17:15,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742080_1256 (size=12001) 2024-12-05T05:17:15,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375895962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375895962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375895962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375895962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:15,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:15,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375895973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:15,974 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T05:17:16,253 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e31e47e8efea4a6b9d218ea35c25924d 2024-12-05T05:17:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/6b6ad38f6fa24dc2908aa13893676564 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6b6ad38f6fa24dc2908aa13893676564 2024-12-05T05:17:16,261 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6b6ad38f6fa24dc2908aa13893676564, entries=150, sequenceid=85, filesize=11.7 K 2024-12-05T05:17:16,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c53c0b7cb855419dbeec9bec408659ba as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c53c0b7cb855419dbeec9bec408659ba 2024-12-05T05:17:16,266 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c53c0b7cb855419dbeec9bec408659ba, entries=150, sequenceid=85, filesize=11.7 K 2024-12-05T05:17:16,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e31e47e8efea4a6b9d218ea35c25924d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e31e47e8efea4a6b9d218ea35c25924d 2024-12-05T05:17:16,271 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e31e47e8efea4a6b9d218ea35c25924d, entries=150, sequenceid=85, filesize=11.7 K 2024-12-05T05:17:16,272 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ac783a93e1fe6938afb704b21f8b8675 in 984ms, sequenceid=85, compaction requested=true 2024-12-05T05:17:16,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:16,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:16,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-12-05T05:17:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-12-05T05:17:16,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-05T05:17:16,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2950 sec 2024-12-05T05:17:16,277 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.3030 sec 2024-12-05T05:17:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:16,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T05:17:16,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:16,476 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:16,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/edcedefb69834df3868052970f6ddb69 is 50, key is test_row_0/A:col10/1733375836473/Put/seqid=0 2024-12-05T05:17:16,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742081_1257 (size=12001) 2024-12-05T05:17:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375896486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375896486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375896487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375896487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375896488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375896603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375896603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375896603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375896603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375896603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375896813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375896813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375896813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375896813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375896813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:16,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/edcedefb69834df3868052970f6ddb69 2024-12-05T05:17:16,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/da955ca184d548db92da97be104f63d5 is 50, key is test_row_0/B:col10/1733375836473/Put/seqid=0 2024-12-05T05:17:16,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742082_1258 (size=12001) 2024-12-05T05:17:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375897127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375897127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375897127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375897127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375897127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-12-05T05:17:17,133 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:17,134 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:17,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-12-05T05:17:17,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T05:17:17,137 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:17,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:17,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:17,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T05:17:17,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T05:17:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:17,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:17,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:17,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:17,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:17,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/da955ca184d548db92da97be104f63d5 2024-12-05T05:17:17,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/9332e0e1302045fcb518885951706551 is 50, key is test_row_0/C:col10/1733375836473/Put/seqid=0 2024-12-05T05:17:17,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742083_1259 (size=12001) 2024-12-05T05:17:17,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/9332e0e1302045fcb518885951706551 2024-12-05T05:17:17,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/edcedefb69834df3868052970f6ddb69 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/edcedefb69834df3868052970f6ddb69 2024-12-05T05:17:17,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/edcedefb69834df3868052970f6ddb69, entries=150, sequenceid=104, filesize=11.7 K 2024-12-05T05:17:17,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/da955ca184d548db92da97be104f63d5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da955ca184d548db92da97be104f63d5 2024-12-05T05:17:17,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da955ca184d548db92da97be104f63d5, entries=150, sequenceid=104, filesize=11.7 K 2024-12-05T05:17:17,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/9332e0e1302045fcb518885951706551 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/9332e0e1302045fcb518885951706551 2024-12-05T05:17:17,382 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/9332e0e1302045fcb518885951706551, entries=150, sequenceid=104, filesize=11.7 K 2024-12-05T05:17:17,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ac783a93e1fe6938afb704b21f8b8675 in 909ms, sequenceid=104, compaction requested=true 2024-12-05T05:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:17,383 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:17,383 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:17,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:17,386 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:17,386 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:17,386 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:17,386 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:17,386 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:17,386 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:17,386 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/313136bebb3a4b37832e8b9cc8a7e9e1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/fbe6715473894acd88da9865ee84ae40, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6b6ad38f6fa24dc2908aa13893676564, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/edcedefb69834df3868052970f6ddb69] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=47.0 K 2024-12-05T05:17:17,386 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/70265c04bce544dc8bfdfcf1788840ca, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8e0c2e93e62b480f8481b1a556414098, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c53c0b7cb855419dbeec9bec408659ba, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da955ca184d548db92da97be104f63d5] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=47.0 K 2024-12-05T05:17:17,386 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 313136bebb3a4b37832e8b9cc8a7e9e1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375834130 2024-12-05T05:17:17,386 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 70265c04bce544dc8bfdfcf1788840ca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375834130 2024-12-05T05:17:17,387 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting fbe6715473894acd88da9865ee84ae40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733375834288 2024-12-05T05:17:17,387 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e0c2e93e62b480f8481b1a556414098, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733375834288 2024-12-05T05:17:17,387 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b6ad38f6fa24dc2908aa13893676564, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733375834653 2024-12-05T05:17:17,387 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c53c0b7cb855419dbeec9bec408659ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733375834653 2024-12-05T05:17:17,388 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting edcedefb69834df3868052970f6ddb69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733375835317 2024-12-05T05:17:17,393 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting da955ca184d548db92da97be104f63d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733375835317 2024-12-05T05:17:17,428 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:17,428 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/d6140543c9dd498896c50f1fed975f51 is 50, key is test_row_0/A:col10/1733375836473/Put/seqid=0 2024-12-05T05:17:17,439 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#214 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:17,440 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/28f26a7c7894437ba7ba290eaa8964fc is 50, key is test_row_0/B:col10/1733375836473/Put/seqid=0 2024-12-05T05:17:17,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T05:17:17,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-12-05T05:17:17,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:17,459 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:17:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:17,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:17,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742084_1260 (size=12241) 2024-12-05T05:17:17,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742085_1261 (size=12241) 2024-12-05T05:17:17,482 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/d6140543c9dd498896c50f1fed975f51 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d6140543c9dd498896c50f1fed975f51 2024-12-05T05:17:17,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/035a9f300c1d49d390fe78e42b9ec610 is 50, key is test_row_0/A:col10/1733375836487/Put/seqid=0 2024-12-05T05:17:17,502 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/28f26a7c7894437ba7ba290eaa8964fc as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/28f26a7c7894437ba7ba290eaa8964fc 2024-12-05T05:17:17,502 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into d6140543c9dd498896c50f1fed975f51(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:17,502 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:17,502 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=12, startTime=1733375837383; duration=0sec 2024-12-05T05:17:17,502 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:17,503 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:17,503 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:17,504 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:17,504 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:17,504 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:17,504 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/42621997bed64497a22994f79c093dd0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d86a9ecd60a645f2a7b9e8f55fae5546, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e31e47e8efea4a6b9d218ea35c25924d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/9332e0e1302045fcb518885951706551] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=47.0 K 2024-12-05T05:17:17,505 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42621997bed64497a22994f79c093dd0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375834130 2024-12-05T05:17:17,505 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting d86a9ecd60a645f2a7b9e8f55fae5546, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1733375834288 2024-12-05T05:17:17,505 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e31e47e8efea4a6b9d218ea35c25924d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1733375834653 2024-12-05T05:17:17,506 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9332e0e1302045fcb518885951706551, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733375835317 2024-12-05T05:17:17,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742086_1262 (size=12001) 2024-12-05T05:17:17,507 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/035a9f300c1d49d390fe78e42b9ec610 2024-12-05T05:17:17,510 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 28f26a7c7894437ba7ba290eaa8964fc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:17,510 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:17,510 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=12, startTime=1733375837383; duration=0sec 2024-12-05T05:17:17,510 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:17,510 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c9bb787550e2418faa107deb5f777e10 is 50, key is test_row_0/B:col10/1733375836487/Put/seqid=0 2024-12-05T05:17:17,523 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:17,524 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/97bc242f5c314ae7b43a1700cbe09631 is 50, key is test_row_0/C:col10/1733375836473/Put/seqid=0 2024-12-05T05:17:17,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742087_1263 (size=12001) 2024-12-05T05:17:17,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742088_1264 (size=12241) 2024-12-05T05:17:17,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:17,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:17,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375897661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375897663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375897668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375897668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375897668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T05:17:17,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375897773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375897773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375897782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375897783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375897783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,939 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c9bb787550e2418faa107deb5f777e10 2024-12-05T05:17:17,948 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/97bc242f5c314ae7b43a1700cbe09631 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97bc242f5c314ae7b43a1700cbe09631 2024-12-05T05:17:17,957 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into 97bc242f5c314ae7b43a1700cbe09631(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:17,957 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:17,957 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=12, startTime=1733375837383; duration=0sec 2024-12-05T05:17:17,957 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:17,957 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/059aba8cab964060a64a1c7cc75f8cce is 50, key is test_row_0/C:col10/1733375836487/Put/seqid=0 2024-12-05T05:17:17,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742089_1265 (size=12001) 2024-12-05T05:17:17,983 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/059aba8cab964060a64a1c7cc75f8cce 2024-12-05T05:17:17,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375897983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375897983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/035a9f300c1d49d390fe78e42b9ec610 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/035a9f300c1d49d390fe78e42b9ec610 2024-12-05T05:17:17,994 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/035a9f300c1d49d390fe78e42b9ec610, entries=150, sequenceid=117, filesize=11.7 K 2024-12-05T05:17:17,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375897993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c9bb787550e2418faa107deb5f777e10 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9bb787550e2418faa107deb5f777e10 2024-12-05T05:17:17,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:17,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375897993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375897993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,002 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9bb787550e2418faa107deb5f777e10, entries=150, sequenceid=117, filesize=11.7 K 2024-12-05T05:17:18,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/059aba8cab964060a64a1c7cc75f8cce as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/059aba8cab964060a64a1c7cc75f8cce 2024-12-05T05:17:18,011 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/059aba8cab964060a64a1c7cc75f8cce, entries=150, sequenceid=117, filesize=11.7 K 2024-12-05T05:17:18,012 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ac783a93e1fe6938afb704b21f8b8675 in 554ms, sequenceid=117, compaction requested=false 2024-12-05T05:17:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-12-05T05:17:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-12-05T05:17:18,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-12-05T05:17:18,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 880 msec 2024-12-05T05:17:18,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 896 msec 2024-12-05T05:17:18,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-05T05:17:18,273 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:18,273 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:18,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-12-05T05:17:18,275 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:18,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T05:17:18,277 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:18,277 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:18,297 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T05:17:18,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:18,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375837649, max timestamp=1733375838295Num uniques 859; 2024-12-05T05:17:18,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375837649, max timestamp=1733375838295Num uniques 859; 2024-12-05T05:17:18,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:18,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:18,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:18,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/cbca17cd7a0b416487f782c522df7aca is 50, key is test_row_0/A:col10/1733375838295/Put/seqid=0 2024-12-05T05:17:18,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742090_1266 (size=14541) 2024-12-05T05:17:18,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/cbca17cd7a0b416487f782c522df7aca 2024-12-05T05:17:18,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375898330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375838328, max timestamp=1733375838336Num uniques 859; 2024-12-05T05:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375838328, max timestamp=1733375838336Num uniques 859; 2024-12-05T05:17:18,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375838328, max timestamp=1733375838336Num uniques 859; 2024-12-05T05:17:18,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:18,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:18,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/1052c79f55444d66b3a11b139faef833 is 50, key is test_row_0/B:col10/1733375838295/Put/seqid=0 2024-12-05T05:17:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375898335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742091_1267 (size=12151) 2024-12-05T05:17:18,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/1052c79f55444d66b3a11b139faef833 2024-12-05T05:17:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T05:17:18,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e37f20149638410bb47e9782fff09b52 is 50, key is test_row_0/C:col10/1733375838295/Put/seqid=0 2024-12-05T05:17:18,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375898405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375898413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375898415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742092_1268 (size=12151) 2024-12-05T05:17:18,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T05:17:18,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:18,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375898460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375898465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375898522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375898523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375898533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T05:17:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T05:17:18,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375898673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375898673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375898733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375898733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375898743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:18,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:18,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T05:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,748 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:18,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e37f20149638410bb47e9782fff09b52 2024-12-05T05:17:18,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/cbca17cd7a0b416487f782c522df7aca as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/cbca17cd7a0b416487f782c522df7aca 2024-12-05T05:17:18,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/cbca17cd7a0b416487f782c522df7aca, entries=200, sequenceid=142, filesize=14.2 K 2024-12-05T05:17:18,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/1052c79f55444d66b3a11b139faef833 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/1052c79f55444d66b3a11b139faef833 2024-12-05T05:17:18,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/1052c79f55444d66b3a11b139faef833, entries=150, sequenceid=142, filesize=11.9 K 2024-12-05T05:17:18,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e37f20149638410bb47e9782fff09b52 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e37f20149638410bb47e9782fff09b52 2024-12-05T05:17:18,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e37f20149638410bb47e9782fff09b52, entries=150, sequenceid=142, filesize=11.9 K 2024-12-05T05:17:18,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~184.05 KB/188464, currentSize=120.76 KB/123660 for ac783a93e1fe6938afb704b21f8b8675 in 562ms, sequenceid=142, compaction requested=true 2024-12-05T05:17:18,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:18,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:18,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:18,860 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:18,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:18,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:18,861 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:18,861 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:18,861 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,861 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d6140543c9dd498896c50f1fed975f51, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/035a9f300c1d49d390fe78e42b9ec610, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/cbca17cd7a0b416487f782c522df7aca] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=37.9 K 2024-12-05T05:17:18,863 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:18,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:18,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:18,869 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6140543c9dd498896c50f1fed975f51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733375835317 2024-12-05T05:17:18,871 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 035a9f300c1d49d390fe78e42b9ec610, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733375836485 2024-12-05T05:17:18,872 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbca17cd7a0b416487f782c522df7aca, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733375837665 2024-12-05T05:17:18,872 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:18,872 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:18,872 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,872 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/28f26a7c7894437ba7ba290eaa8964fc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9bb787550e2418faa107deb5f777e10, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/1052c79f55444d66b3a11b139faef833] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=35.5 K 2024-12-05T05:17:18,875 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 28f26a7c7894437ba7ba290eaa8964fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733375835317 2024-12-05T05:17:18,876 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c9bb787550e2418faa107deb5f777e10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733375836485 2024-12-05T05:17:18,876 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 1052c79f55444d66b3a11b139faef833, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733375837665 2024-12-05T05:17:18,890 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:18,891 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/da7fc6d2e3c043b2b095fbc7d3137b3e is 50, key is test_row_0/B:col10/1733375838295/Put/seqid=0 2024-12-05T05:17:18,898 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:18,898 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/d997c3a6ff4f42ffabe9727d38bf926e is 50, key is test_row_0/A:col10/1733375838295/Put/seqid=0 2024-12-05T05:17:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T05:17:18,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-12-05T05:17:18,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,907 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-12-05T05:17:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:18,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:18,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742094_1270 (size=12493) 2024-12-05T05:17:18,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742093_1269 (size=12493) 2024-12-05T05:17:18,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/ef44aa40f8f640d8bda863f2f02147a0 is 50, key is test_row_0/A:col10/1733375838413/Put/seqid=0 2024-12-05T05:17:18,919 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/d997c3a6ff4f42ffabe9727d38bf926e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d997c3a6ff4f42ffabe9727d38bf926e 2024-12-05T05:17:18,924 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into d997c3a6ff4f42ffabe9727d38bf926e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:18,924 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:18,924 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=13, startTime=1733375838859; duration=0sec 2024-12-05T05:17:18,924 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:18,924 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:18,925 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:18,926 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:18,926 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:18,926 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:18,927 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97bc242f5c314ae7b43a1700cbe09631, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/059aba8cab964060a64a1c7cc75f8cce, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e37f20149638410bb47e9782fff09b52] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=35.5 K 2024-12-05T05:17:18,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742095_1271 (size=12151) 2024-12-05T05:17:18,934 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97bc242f5c314ae7b43a1700cbe09631, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1733375835317 2024-12-05T05:17:18,935 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 059aba8cab964060a64a1c7cc75f8cce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1733375836485 2024-12-05T05:17:18,936 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e37f20149638410bb47e9782fff09b52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733375837665 2024-12-05T05:17:18,950 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/ef44aa40f8f640d8bda863f2f02147a0 2024-12-05T05:17:18,962 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:18,963 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/1952548d9bb34aada1afe0bb629011fb is 50, key is test_row_0/C:col10/1733375838295/Put/seqid=0 2024-12-05T05:17:18,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742096_1272 (size=12493) 2024-12-05T05:17:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:18,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:18,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/40d5b21e7ad74826b7d3e913461b3e9b is 50, key is test_row_0/B:col10/1733375838413/Put/seqid=0 2024-12-05T05:17:18,996 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/1952548d9bb34aada1afe0bb629011fb as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/1952548d9bb34aada1afe0bb629011fb 2024-12-05T05:17:19,003 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into 1952548d9bb34aada1afe0bb629011fb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:19,003 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:19,003 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=13, startTime=1733375838860; duration=0sec 2024-12-05T05:17:19,003 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:19,003 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:19,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742097_1273 (size=12151) 2024-12-05T05:17:19,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375899023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375899025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375899044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375899045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375899056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375899132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375899133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,317 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/da7fc6d2e3c043b2b095fbc7d3137b3e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da7fc6d2e3c043b2b095fbc7d3137b3e 2024-12-05T05:17:19,324 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into da7fc6d2e3c043b2b095fbc7d3137b3e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:19,324 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:19,324 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=13, startTime=1733375838860; duration=0sec 2024-12-05T05:17:19,324 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:19,324 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:19,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375899342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375899342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T05:17:19,414 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/40d5b21e7ad74826b7d3e913461b3e9b 2024-12-05T05:17:19,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ebd9df7432a64ed68dab791d34ed26ae is 50, key is test_row_0/C:col10/1733375838413/Put/seqid=0 2024-12-05T05:17:19,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742098_1274 (size=12151) 2024-12-05T05:17:19,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375899552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375899563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375899573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375899653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375899653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:19,835 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ebd9df7432a64ed68dab791d34ed26ae 2024-12-05T05:17:19,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/ef44aa40f8f640d8bda863f2f02147a0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ef44aa40f8f640d8bda863f2f02147a0 2024-12-05T05:17:19,845 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ef44aa40f8f640d8bda863f2f02147a0, entries=150, sequenceid=177, filesize=11.9 K 2024-12-05T05:17:19,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/40d5b21e7ad74826b7d3e913461b3e9b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/40d5b21e7ad74826b7d3e913461b3e9b 2024-12-05T05:17:19,850 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/40d5b21e7ad74826b7d3e913461b3e9b, entries=150, sequenceid=177, filesize=11.9 K 2024-12-05T05:17:19,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ebd9df7432a64ed68dab791d34ed26ae as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ebd9df7432a64ed68dab791d34ed26ae 2024-12-05T05:17:19,855 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ebd9df7432a64ed68dab791d34ed26ae, entries=150, sequenceid=177, filesize=11.9 K 2024-12-05T05:17:19,857 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for ac783a93e1fe6938afb704b21f8b8675 in 949ms, sequenceid=177, compaction requested=false 2024-12-05T05:17:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:19,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-12-05T05:17:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-12-05T05:17:19,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-05T05:17:19,860 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5810 sec 2024-12-05T05:17:19,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.5910 sec 2024-12-05T05:17:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:20,165 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:17:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:20,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:20,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/5331bf6c296c4bc196c6ec176250752b is 50, key is test_row_0/A:col10/1733375839015/Put/seqid=0 2024-12-05T05:17:20,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742099_1275 (size=12151) 2024-12-05T05:17:20,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375900200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375900200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375900313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375900313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-12-05T05:17:20,424 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:20,424 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:20,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-05T05:17:20,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T05:17:20,426 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:20,427 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:20,427 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:20,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375900523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375900523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T05:17:20,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375900562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,564 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T05:17:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:20,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375900583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375900583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,584 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,585 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,585 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/5331bf6c296c4bc196c6ec176250752b 2024-12-05T05:17:20,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/d5fe6b2b0e5d4dba9947f0a9c12f1722 is 50, key is test_row_0/B:col10/1733375839015/Put/seqid=0 2024-12-05T05:17:20,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742100_1276 (size=12151) 2024-12-05T05:17:20,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T05:17:20,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:20,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:20,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:20,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T05:17:20,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375900833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,834 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375900833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:20,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:20,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T05:17:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:20,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:20,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:20,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:17:20,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T05:17:21,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/d5fe6b2b0e5d4dba9947f0a9c12f1722 2024-12-05T05:17:21,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/58f4c2124cda42088ab8be8860da1184 is 50, key is test_row_0/C:col10/1733375839015/Put/seqid=0 2024-12-05T05:17:21,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742101_1277 (size=12151) 2024-12-05T05:17:21,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T05:17:21,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:21,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T05:17:21,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T05:17:21,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:21,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375901343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:21,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:21,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375901343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:21,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:21,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:21,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T05:17:21,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:21,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/58f4c2124cda42088ab8be8860da1184 2024-12-05T05:17:21,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/5331bf6c296c4bc196c6ec176250752b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5331bf6c296c4bc196c6ec176250752b 2024-12-05T05:17:21,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5331bf6c296c4bc196c6ec176250752b, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T05:17:21,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/d5fe6b2b0e5d4dba9947f0a9c12f1722 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d5fe6b2b0e5d4dba9947f0a9c12f1722 2024-12-05T05:17:21,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d5fe6b2b0e5d4dba9947f0a9c12f1722, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T05:17:21,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/58f4c2124cda42088ab8be8860da1184 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/58f4c2124cda42088ab8be8860da1184 2024-12-05T05:17:21,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/58f4c2124cda42088ab8be8860da1184, entries=150, sequenceid=193, filesize=11.9 K 2024-12-05T05:17:21,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for ac783a93e1fe6938afb704b21f8b8675 in 1295ms, sequenceid=193, compaction requested=true 2024-12-05T05:17:21,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:21,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:21,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:21,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:21,460 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:21,461 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T05:17:21,461 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:21,462 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:21,462 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:21,462 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,462 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:21,462 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:21,462 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d997c3a6ff4f42ffabe9727d38bf926e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ef44aa40f8f640d8bda863f2f02147a0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5331bf6c296c4bc196c6ec176250752b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=35.9 K 2024-12-05T05:17:21,462 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,462 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da7fc6d2e3c043b2b095fbc7d3137b3e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/40d5b21e7ad74826b7d3e913461b3e9b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d5fe6b2b0e5d4dba9947f0a9c12f1722] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=35.9 K 2024-12-05T05:17:21,462 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d997c3a6ff4f42ffabe9727d38bf926e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733375837665 2024-12-05T05:17:21,463 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7fc6d2e3c043b2b095fbc7d3137b3e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733375837665 2024-12-05T05:17:21,463 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ef44aa40f8f640d8bda863f2f02147a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733375838400 2024-12-05T05:17:21,463 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40d5b21e7ad74826b7d3e913461b3e9b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733375838400 2024-12-05T05:17:21,463 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 5331bf6c296c4bc196c6ec176250752b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733375839015 2024-12-05T05:17:21,463 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5fe6b2b0e5d4dba9947f0a9c12f1722, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733375839015 2024-12-05T05:17:21,481 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:21,482 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/8286231e4fa543dcb95867e3ddd6cc3d is 50, key is test_row_0/B:col10/1733375839015/Put/seqid=0 2024-12-05T05:17:21,482 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:21,486 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/4e0b4c6a184d43128f4b6f901dcacf97 is 50, key is test_row_0/A:col10/1733375839015/Put/seqid=0 2024-12-05T05:17:21,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742102_1278 (size=12595) 2024-12-05T05:17:21,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-05T05:17:21,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,506 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T05:17:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:21,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:21,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742103_1279 (size=12595) 2024-12-05T05:17:21,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/0325d7f41da14ff591ac8fe1c7ae6268 is 50, key is test_row_0/A:col10/1733375840185/Put/seqid=0 2024-12-05T05:17:21,527 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/4e0b4c6a184d43128f4b6f901dcacf97 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/4e0b4c6a184d43128f4b6f901dcacf97 2024-12-05T05:17:21,536 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into 4e0b4c6a184d43128f4b6f901dcacf97(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:21,536 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:21,536 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=13, startTime=1733375841460; duration=0sec 2024-12-05T05:17:21,536 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:21,536 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:21,536 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:21,537 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:21,537 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:21,538 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:21,538 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/1952548d9bb34aada1afe0bb629011fb, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ebd9df7432a64ed68dab791d34ed26ae, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/58f4c2124cda42088ab8be8860da1184] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=35.9 K 2024-12-05T05:17:21,538 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 1952548d9bb34aada1afe0bb629011fb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1733375837665 2024-12-05T05:17:21,539 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd9df7432a64ed68dab791d34ed26ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1733375838400 2024-12-05T05:17:21,539 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 58f4c2124cda42088ab8be8860da1184, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733375839015 2024-12-05T05:17:21,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742104_1280 (size=12151) 2024-12-05T05:17:21,541 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/0325d7f41da14ff591ac8fe1c7ae6268 2024-12-05T05:17:21,551 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:21,554 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/c6d93bd62ec54cbe8b9fe149e921d847 is 50, key is test_row_0/C:col10/1733375839015/Put/seqid=0 2024-12-05T05:17:21,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/6332144b00814784a4c6d1b4e54af53d is 50, key is test_row_0/B:col10/1733375840185/Put/seqid=0 2024-12-05T05:17:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T05:17:21,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742106_1282 (size=12151) 2024-12-05T05:17:21,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742105_1281 (size=12595) 2024-12-05T05:17:21,909 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/8286231e4fa543dcb95867e3ddd6cc3d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8286231e4fa543dcb95867e3ddd6cc3d 2024-12-05T05:17:21,915 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 8286231e4fa543dcb95867e3ddd6cc3d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:21,915 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:21,915 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=13, startTime=1733375841460; duration=0sec 2024-12-05T05:17:21,916 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:21,916 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:21,965 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/6332144b00814784a4c6d1b4e54af53d 2024-12-05T05:17:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/c4ac8790c0d542dbbb99b2d78bea7309 is 50, key is test_row_0/C:col10/1733375840185/Put/seqid=0 2024-12-05T05:17:21,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742107_1283 (size=12151) 2024-12-05T05:17:21,987 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/c6d93bd62ec54cbe8b9fe149e921d847 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c6d93bd62ec54cbe8b9fe149e921d847 2024-12-05T05:17:22,003 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into c6d93bd62ec54cbe8b9fe149e921d847(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:22,003 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:22,003 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=13, startTime=1733375841461; duration=0sec 2024-12-05T05:17:22,003 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:22,003 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:22,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:22,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375902365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375902368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,385 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/c4ac8790c0d542dbbb99b2d78bea7309 2024-12-05T05:17:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/0325d7f41da14ff591ac8fe1c7ae6268 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/0325d7f41da14ff591ac8fe1c7ae6268 2024-12-05T05:17:22,392 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/0325d7f41da14ff591ac8fe1c7ae6268, entries=150, sequenceid=213, filesize=11.9 K 2024-12-05T05:17:22,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/6332144b00814784a4c6d1b4e54af53d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6332144b00814784a4c6d1b4e54af53d 2024-12-05T05:17:22,396 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6332144b00814784a4c6d1b4e54af53d, entries=150, sequenceid=213, filesize=11.9 K 2024-12-05T05:17:22,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/c4ac8790c0d542dbbb99b2d78bea7309 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c4ac8790c0d542dbbb99b2d78bea7309 2024-12-05T05:17:22,401 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c4ac8790c0d542dbbb99b2d78bea7309, entries=150, sequenceid=213, filesize=11.9 K 2024-12-05T05:17:22,404 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for ac783a93e1fe6938afb704b21f8b8675 in 898ms, sequenceid=213, compaction requested=false 2024-12-05T05:17:22,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:22,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-05T05:17:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-05T05:17:22,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-05T05:17:22,409 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9800 sec 2024-12-05T05:17:22,411 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.9860 sec 2024-12-05T05:17:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:22,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:17:22,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:22,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:22,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:22,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:22,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:22,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:22,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/9a2c8ee9e8c945fa9071361cc285d7ab is 50, key is test_row_0/A:col10/1733375842473/Put/seqid=0 2024-12-05T05:17:22,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742108_1284 (size=12151) 2024-12-05T05:17:22,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/9a2c8ee9e8c945fa9071361cc285d7ab 2024-12-05T05:17:22,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/08ea1ec8dd6f45cb8aee547ebbae97d2 is 50, key is test_row_0/B:col10/1733375842473/Put/seqid=0 2024-12-05T05:17:22,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742109_1285 (size=12151) 2024-12-05T05:17:22,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/08ea1ec8dd6f45cb8aee547ebbae97d2 2024-12-05T05:17:22,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375902523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375902526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e54f7c54c86944e099e7f6f0f2ccbe12 is 50, key is test_row_0/C:col10/1733375842473/Put/seqid=0 2024-12-05T05:17:22,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742110_1286 (size=12151) 2024-12-05T05:17:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-05T05:17:22,573 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:22,574 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:22,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-05T05:17:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375902572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,575 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T05:17:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,577 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:22,577 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:22,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:22,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375902593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,595 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:22,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375902603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,606 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375902633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375902643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T05:17:22,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T05:17:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:22,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:22,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:22,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:22,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375902843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,846 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:22,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375902852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:22,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:22,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T05:17:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:22,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:22,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:22,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:22,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:22,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T05:17:22,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e54f7c54c86944e099e7f6f0f2ccbe12 2024-12-05T05:17:22,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/9a2c8ee9e8c945fa9071361cc285d7ab as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/9a2c8ee9e8c945fa9071361cc285d7ab 2024-12-05T05:17:22,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/9a2c8ee9e8c945fa9071361cc285d7ab, entries=150, sequenceid=229, filesize=11.9 K 2024-12-05T05:17:22,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/08ea1ec8dd6f45cb8aee547ebbae97d2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/08ea1ec8dd6f45cb8aee547ebbae97d2 2024-12-05T05:17:22,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/08ea1ec8dd6f45cb8aee547ebbae97d2, entries=150, sequenceid=229, filesize=11.9 K 2024-12-05T05:17:22,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e54f7c54c86944e099e7f6f0f2ccbe12 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e54f7c54c86944e099e7f6f0f2ccbe12 2024-12-05T05:17:22,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e54f7c54c86944e099e7f6f0f2ccbe12, entries=150, sequenceid=229, filesize=11.9 K 2024-12-05T05:17:22,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ac783a93e1fe6938afb704b21f8b8675 in 513ms, sequenceid=229, compaction requested=true 2024-12-05T05:17:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:22,987 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:22,987 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:22,988 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:22,988 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:22,988 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:22,988 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:22,988 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:22,988 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:22,988 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8286231e4fa543dcb95867e3ddd6cc3d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6332144b00814784a4c6d1b4e54af53d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/08ea1ec8dd6f45cb8aee547ebbae97d2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.0 K 2024-12-05T05:17:22,988 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/4e0b4c6a184d43128f4b6f901dcacf97, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/0325d7f41da14ff591ac8fe1c7ae6268, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/9a2c8ee9e8c945fa9071361cc285d7ab] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.0 K 2024-12-05T05:17:22,988 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8286231e4fa543dcb95867e3ddd6cc3d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733375839015 2024-12-05T05:17:22,988 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e0b4c6a184d43128f4b6f901dcacf97, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733375839015 2024-12-05T05:17:22,989 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6332144b00814784a4c6d1b4e54af53d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733375840185 2024-12-05T05:17:22,989 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0325d7f41da14ff591ac8fe1c7ae6268, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733375840185 2024-12-05T05:17:22,989 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 08ea1ec8dd6f45cb8aee547ebbae97d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375842358 2024-12-05T05:17:22,989 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a2c8ee9e8c945fa9071361cc285d7ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375842358 2024-12-05T05:17:22,995 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:22,995 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/303fbc2aca1b4e12bdf3aa677e59ecaf is 50, key is test_row_0/B:col10/1733375842473/Put/seqid=0 2024-12-05T05:17:22,996 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#241 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:22,997 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f585fb3780e543239788d8bc9299d2f3 is 50, key is test_row_0/A:col10/1733375842473/Put/seqid=0 2024-12-05T05:17:23,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742111_1287 (size=12697) 2024-12-05T05:17:23,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742112_1288 (size=12697) 2024-12-05T05:17:23,022 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f585fb3780e543239788d8bc9299d2f3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f585fb3780e543239788d8bc9299d2f3 2024-12-05T05:17:23,026 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into f585fb3780e543239788d8bc9299d2f3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:23,026 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:23,026 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=13, startTime=1733375842987; duration=0sec 2024-12-05T05:17:23,026 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:23,026 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:23,027 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:23,027 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:23,027 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:23,027 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:23,028 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c6d93bd62ec54cbe8b9fe149e921d847, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c4ac8790c0d542dbbb99b2d78bea7309, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e54f7c54c86944e099e7f6f0f2ccbe12] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.0 K 2024-12-05T05:17:23,028 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d93bd62ec54cbe8b9fe149e921d847, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1733375839015 2024-12-05T05:17:23,028 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4ac8790c0d542dbbb99b2d78bea7309, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1733375840185 2024-12-05T05:17:23,028 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e54f7c54c86944e099e7f6f0f2ccbe12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375842358 2024-12-05T05:17:23,035 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:23,035 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/6a707db7706a42aa89f01564345fe355 is 50, key is test_row_0/C:col10/1733375842473/Put/seqid=0 2024-12-05T05:17:23,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-05T05:17:23,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:23,036 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:17:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:23,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:23,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f01fb7b0a1a143729eb9236558a41320 is 50, key is test_row_0/A:col10/1733375842524/Put/seqid=0 2024-12-05T05:17:23,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742113_1289 (size=12697) 2024-12-05T05:17:23,046 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/6a707db7706a42aa89f01564345fe355 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/6a707db7706a42aa89f01564345fe355 2024-12-05T05:17:23,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742114_1290 (size=12151) 2024-12-05T05:17:23,050 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into 6a707db7706a42aa89f01564345fe355(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:23,050 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:23,050 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=13, startTime=1733375842987; duration=0sec 2024-12-05T05:17:23,051 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:23,051 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:23,052 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f01fb7b0a1a143729eb9236558a41320 2024-12-05T05:17:23,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/04d4bff0e71c4472a5616a4304c3601e is 50, key is test_row_0/B:col10/1733375842524/Put/seqid=0 2024-12-05T05:17:23,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742115_1291 (size=12151) 2024-12-05T05:17:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:23,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:23,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375903173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:23,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375903175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:23,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T05:17:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:23,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375903283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:23,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375903284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:23,422 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/303fbc2aca1b4e12bdf3aa677e59ecaf as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/303fbc2aca1b4e12bdf3aa677e59ecaf 2024-12-05T05:17:23,427 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 303fbc2aca1b4e12bdf3aa677e59ecaf(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:23,427 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:23,427 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=13, startTime=1733375842987; duration=0sec 2024-12-05T05:17:23,427 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:23,427 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:23,463 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/04d4bff0e71c4472a5616a4304c3601e 2024-12-05T05:17:23,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/388da213f1874d2190afe3317598c09b is 50, key is test_row_0/C:col10/1733375842524/Put/seqid=0 2024-12-05T05:17:23,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742116_1292 (size=12151) 2024-12-05T05:17:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:23,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375903492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:23,494 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:23,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375903492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T05:17:23,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375903803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375903803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:23,874 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/388da213f1874d2190afe3317598c09b 2024-12-05T05:17:23,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f01fb7b0a1a143729eb9236558a41320 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f01fb7b0a1a143729eb9236558a41320 2024-12-05T05:17:23,883 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f01fb7b0a1a143729eb9236558a41320, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T05:17:23,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/04d4bff0e71c4472a5616a4304c3601e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/04d4bff0e71c4472a5616a4304c3601e 2024-12-05T05:17:23,888 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/04d4bff0e71c4472a5616a4304c3601e, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T05:17:23,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/388da213f1874d2190afe3317598c09b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/388da213f1874d2190afe3317598c09b 2024-12-05T05:17:23,892 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/388da213f1874d2190afe3317598c09b, entries=150, sequenceid=249, filesize=11.9 K 2024-12-05T05:17:23,893 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ac783a93e1fe6938afb704b21f8b8675 in 857ms, sequenceid=249, compaction requested=false 2024-12-05T05:17:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:23,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-05T05:17:23,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-05T05:17:23,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-05T05:17:23,896 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3170 sec 2024-12-05T05:17:23,897 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.3220 sec 2024-12-05T05:17:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:24,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T05:17:24,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:24,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:24,315 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:24,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:24,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/aef3449dfb744a92a9f8cddff493946b is 50, key is test_row_0/A:col10/1733375844314/Put/seqid=0 2024-12-05T05:17:24,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742117_1293 (size=17181) 2024-12-05T05:17:24,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375904344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:24,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375904344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:24,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375904453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:24,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375904453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:24,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375904663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375904663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,666 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-05T05:17:24,723 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:24,724 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:24,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/aef3449dfb744a92a9f8cddff493946b 2024-12-05T05:17:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-05T05:17:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T05:17:24,730 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:24,734 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:24,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:24,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/7d5426a2cbb0422895dc4551133df550 is 50, key is test_row_0/B:col10/1733375844314/Put/seqid=0 2024-12-05T05:17:24,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742118_1294 (size=12301) 2024-12-05T05:17:24,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/7d5426a2cbb0422895dc4551133df550 2024-12-05T05:17:24,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e55f44cc77994a6e959d9cbc2743fc9f is 50, key is test_row_0/C:col10/1733375844314/Put/seqid=0 2024-12-05T05:17:24,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742119_1295 (size=12301) 2024-12-05T05:17:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T05:17:24,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T05:17:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:24,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:24,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:24,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375904973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375904973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T05:17:25,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T05:17:25,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:25,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:25,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:25,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:25,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:25,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e55f44cc77994a6e959d9cbc2743fc9f 2024-12-05T05:17:25,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/aef3449dfb744a92a9f8cddff493946b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/aef3449dfb744a92a9f8cddff493946b 2024-12-05T05:17:25,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/aef3449dfb744a92a9f8cddff493946b, entries=250, sequenceid=266, filesize=16.8 K 2024-12-05T05:17:25,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/7d5426a2cbb0422895dc4551133df550 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7d5426a2cbb0422895dc4551133df550 2024-12-05T05:17:25,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7d5426a2cbb0422895dc4551133df550, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T05:17:25,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/e55f44cc77994a6e959d9cbc2743fc9f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e55f44cc77994a6e959d9cbc2743fc9f 2024-12-05T05:17:25,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T05:17:25,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:25,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:25,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e55f44cc77994a6e959d9cbc2743fc9f, entries=150, sequenceid=266, filesize=12.0 K 2024-12-05T05:17:25,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:25,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:25,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:25,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:25,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for ac783a93e1fe6938afb704b21f8b8675 in 884ms, sequenceid=266, compaction requested=true 2024-12-05T05:17:25,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:25,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:25,200 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:25,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:25,200 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:25,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:25,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:25,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:25,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:25,201 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42029 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:25,201 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:25,201 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:25,201 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f585fb3780e543239788d8bc9299d2f3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f01fb7b0a1a143729eb9236558a41320, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/aef3449dfb744a92a9f8cddff493946b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=41.0 K 2024-12-05T05:17:25,201 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f585fb3780e543239788d8bc9299d2f3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375842358 2024-12-05T05:17:25,202 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:25,202 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f01fb7b0a1a143729eb9236558a41320, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733375842514 2024-12-05T05:17:25,202 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:25,202 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:25,202 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/303fbc2aca1b4e12bdf3aa677e59ecaf, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/04d4bff0e71c4472a5616a4304c3601e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7d5426a2cbb0422895dc4551133df550] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.3 K 2024-12-05T05:17:25,202 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting aef3449dfb744a92a9f8cddff493946b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375843167 2024-12-05T05:17:25,202 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 303fbc2aca1b4e12bdf3aa677e59ecaf, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375842358 2024-12-05T05:17:25,202 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 04d4bff0e71c4472a5616a4304c3601e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733375842514 2024-12-05T05:17:25,203 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d5426a2cbb0422895dc4551133df550, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375843174 2024-12-05T05:17:25,210 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:25,211 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/7f96737bcb1a4d7ea32334e594016d3e is 50, key is test_row_0/A:col10/1733375844314/Put/seqid=0 2024-12-05T05:17:25,225 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:25,226 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/6335907b59e94128ab8dd8341ec5ee3d is 50, key is test_row_0/B:col10/1733375844314/Put/seqid=0 2024-12-05T05:17:25,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742121_1297 (size=12949) 2024-12-05T05:17:25,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742120_1296 (size=12949) 2024-12-05T05:17:25,247 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/6335907b59e94128ab8dd8341ec5ee3d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6335907b59e94128ab8dd8341ec5ee3d 2024-12-05T05:17:25,248 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/7f96737bcb1a4d7ea32334e594016d3e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7f96737bcb1a4d7ea32334e594016d3e 2024-12-05T05:17:25,254 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 6335907b59e94128ab8dd8341ec5ee3d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:25,254 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:25,254 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=13, startTime=1733375845200; duration=0sec 2024-12-05T05:17:25,254 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:25,254 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:25,254 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:25,256 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into 7f96737bcb1a4d7ea32334e594016d3e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:25,256 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:25,256 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=13, startTime=1733375845199; duration=0sec 2024-12-05T05:17:25,256 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:25,256 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:25,256 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:25,256 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:25,256 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:25,256 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/6a707db7706a42aa89f01564345fe355, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/388da213f1874d2190afe3317598c09b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e55f44cc77994a6e959d9cbc2743fc9f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.3 K 2024-12-05T05:17:25,257 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a707db7706a42aa89f01564345fe355, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375842358 2024-12-05T05:17:25,257 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 388da213f1874d2190afe3317598c09b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1733375842514 2024-12-05T05:17:25,257 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e55f44cc77994a6e959d9cbc2743fc9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375843174 2024-12-05T05:17:25,265 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:25,266 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/842a7ebd3a6d4696bfa6432ae4813b0a is 50, key is test_row_0/C:col10/1733375844314/Put/seqid=0 2024-12-05T05:17:25,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742122_1298 (size=12949) 2024-12-05T05:17:25,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-05T05:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:25,351 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T05:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:25,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:25,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T05:17:25,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/700c927d77b64b8bac949193c3b7b557 is 50, key is test_row_0/A:col10/1733375844337/Put/seqid=0 2024-12-05T05:17:25,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742123_1299 (size=12301) 2024-12-05T05:17:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:25,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:25,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:25,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375905505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375905504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:25,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:25,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:25,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375905612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375905612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:25,677 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/842a7ebd3a6d4696bfa6432ae4813b0a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/842a7ebd3a6d4696bfa6432ae4813b0a 2024-12-05T05:17:25,681 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into 842a7ebd3a6d4696bfa6432ae4813b0a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:25,681 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:25,681 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=13, startTime=1733375845200; duration=0sec 2024-12-05T05:17:25,681 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:25,681 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:25,759 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/700c927d77b64b8bac949193c3b7b557 2024-12-05T05:17:25,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/16605c3466d74de1bb9f36805990ac3e is 50, key is test_row_0/B:col10/1733375844337/Put/seqid=0 2024-12-05T05:17:25,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742124_1300 (size=12301) 2024-12-05T05:17:25,770 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/16605c3466d74de1bb9f36805990ac3e 2024-12-05T05:17:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/10f5fea1d67f46c19d7cf3eecd0fb80f is 50, key is test_row_0/C:col10/1733375844337/Put/seqid=0 2024-12-05T05:17:25,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742125_1301 (size=12301) 2024-12-05T05:17:25,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375905823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:25,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375905823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:25,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:25,828 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T05:17:26,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375906133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375906133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,182 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/10f5fea1d67f46c19d7cf3eecd0fb80f 2024-12-05T05:17:26,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/700c927d77b64b8bac949193c3b7b557 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/700c927d77b64b8bac949193c3b7b557 2024-12-05T05:17:26,188 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/700c927d77b64b8bac949193c3b7b557, entries=150, sequenceid=285, filesize=12.0 K 2024-12-05T05:17:26,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/16605c3466d74de1bb9f36805990ac3e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/16605c3466d74de1bb9f36805990ac3e 2024-12-05T05:17:26,192 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/16605c3466d74de1bb9f36805990ac3e, entries=150, sequenceid=285, filesize=12.0 K 2024-12-05T05:17:26,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/10f5fea1d67f46c19d7cf3eecd0fb80f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/10f5fea1d67f46c19d7cf3eecd0fb80f 2024-12-05T05:17:26,197 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/10f5fea1d67f46c19d7cf3eecd0fb80f, entries=150, sequenceid=285, filesize=12.0 K 2024-12-05T05:17:26,198 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for ac783a93e1fe6938afb704b21f8b8675 in 847ms, sequenceid=285, compaction requested=false 2024-12-05T05:17:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:26,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-05T05:17:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-05T05:17:26,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-05T05:17:26,201 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-12-05T05:17:26,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.4780 sec 2024-12-05T05:17:26,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:26,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T05:17:26,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:26,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:26,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:26,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:26,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:26,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:26,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/a683775a1f854132820a0b6609bf0272 is 50, key is test_row_0/A:col10/1733375846604/Put/seqid=0 2024-12-05T05:17:26,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742126_1302 (size=14741) 2024-12-05T05:17:26,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/a683775a1f854132820a0b6609bf0272 2024-12-05T05:17:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375906643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375906644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c9853172ecaf4c3bb3d23d8641505138 is 50, key is test_row_0/B:col10/1733375846604/Put/seqid=0 2024-12-05T05:17:26,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375906647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375906647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375906649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742127_1303 (size=12301) 2024-12-05T05:17:26,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c9853172ecaf4c3bb3d23d8641505138 2024-12-05T05:17:26,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/38b22618d56d417cb213c73f2986c4f1 is 50, key is test_row_0/C:col10/1733375846604/Put/seqid=0 2024-12-05T05:17:26,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742128_1304 (size=12301) 2024-12-05T05:17:26,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/38b22618d56d417cb213c73f2986c4f1 2024-12-05T05:17:26,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/a683775a1f854132820a0b6609bf0272 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/a683775a1f854132820a0b6609bf0272 2024-12-05T05:17:26,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/a683775a1f854132820a0b6609bf0272, entries=200, sequenceid=303, filesize=14.4 K 2024-12-05T05:17:26,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c9853172ecaf4c3bb3d23d8641505138 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9853172ecaf4c3bb3d23d8641505138 2024-12-05T05:17:26,737 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9853172ecaf4c3bb3d23d8641505138, entries=150, sequenceid=303, filesize=12.0 K 2024-12-05T05:17:26,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/38b22618d56d417cb213c73f2986c4f1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/38b22618d56d417cb213c73f2986c4f1 2024-12-05T05:17:26,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/38b22618d56d417cb213c73f2986c4f1, entries=150, sequenceid=303, filesize=12.0 K 2024-12-05T05:17:26,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for ac783a93e1fe6938afb704b21f8b8675 in 141ms, sequenceid=303, compaction requested=true 2024-12-05T05:17:26,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:26,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:26,746 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:26,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:26,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:26,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:26,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:26,746 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:26,746 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:26,747 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:26,747 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:26,747 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:26,747 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:26,747 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:26,747 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:26,747 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6335907b59e94128ab8dd8341ec5ee3d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/16605c3466d74de1bb9f36805990ac3e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9853172ecaf4c3bb3d23d8641505138] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.7 K 2024-12-05T05:17:26,747 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7f96737bcb1a4d7ea32334e594016d3e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/700c927d77b64b8bac949193c3b7b557, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/a683775a1f854132820a0b6609bf0272] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=39.1 K 2024-12-05T05:17:26,748 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6335907b59e94128ab8dd8341ec5ee3d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375843174 2024-12-05T05:17:26,748 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f96737bcb1a4d7ea32334e594016d3e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375843174 2024-12-05T05:17:26,749 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 700c927d77b64b8bac949193c3b7b557, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733375844337 2024-12-05T05:17:26,750 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 16605c3466d74de1bb9f36805990ac3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733375844337 2024-12-05T05:17:26,750 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c9853172ecaf4c3bb3d23d8641505138, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733375845496 2024-12-05T05:17:26,750 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a683775a1f854132820a0b6609bf0272, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733375845496 2024-12-05T05:17:26,762 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:26,763 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/0836dbddef6349589582fb4b62557a7d is 50, key is test_row_0/B:col10/1733375846604/Put/seqid=0 2024-12-05T05:17:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:26,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T05:17:26,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:26,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:26,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:26,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:26,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:26,773 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:26,779 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:26,780 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f3e9fc2a099a40639c43dbf903fb9b6b is 50, key is test_row_0/A:col10/1733375846604/Put/seqid=0 2024-12-05T05:17:26,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/47a842f6f9a8474f9799cb8c5c38160c is 50, key is test_row_0/A:col10/1733375846645/Put/seqid=0 2024-12-05T05:17:26,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742129_1305 (size=13051) 2024-12-05T05:17:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375906793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375906796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375906796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742130_1306 (size=13051) 2024-12-05T05:17:26,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742131_1307 (size=12301) 2024-12-05T05:17:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-05T05:17:26,873 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:26,873 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-05T05:17:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T05:17:26,875 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:26,875 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:26,875 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:26,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375906903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375906903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:26,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375906903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:26,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T05:17:27,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T05:17:27,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:27,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375907113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375907113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375907113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T05:17:27,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:27,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T05:17:27,201 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/0836dbddef6349589582fb4b62557a7d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/0836dbddef6349589582fb4b62557a7d 2024-12-05T05:17:27,208 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 0836dbddef6349589582fb4b62557a7d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:27,208 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:27,208 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=13, startTime=1733375846746; duration=0sec 2024-12-05T05:17:27,208 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:27,208 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:27,208 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:27,209 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:27,209 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:27,209 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,209 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/842a7ebd3a6d4696bfa6432ae4813b0a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/10f5fea1d67f46c19d7cf3eecd0fb80f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/38b22618d56d417cb213c73f2986c4f1] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.7 K 2024-12-05T05:17:27,210 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 842a7ebd3a6d4696bfa6432ae4813b0a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733375843174 2024-12-05T05:17:27,210 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 10f5fea1d67f46c19d7cf3eecd0fb80f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1733375844337 2024-12-05T05:17:27,210 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b22618d56d417cb213c73f2986c4f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733375845496 2024-12-05T05:17:27,215 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:27,215 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/b9ff2d7866de4630b99b31a7c1c7b91e is 50, key is test_row_0/C:col10/1733375846604/Put/seqid=0 2024-12-05T05:17:27,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/47a842f6f9a8474f9799cb8c5c38160c 2024-12-05T05:17:27,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742132_1308 (size=13051) 2024-12-05T05:17:27,220 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f3e9fc2a099a40639c43dbf903fb9b6b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f3e9fc2a099a40639c43dbf903fb9b6b 2024-12-05T05:17:27,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/a38e45e40b4d4e3a9033bf9c09f79377 is 50, key is test_row_0/B:col10/1733375846645/Put/seqid=0 2024-12-05T05:17:27,226 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into f3e9fc2a099a40639c43dbf903fb9b6b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:27,226 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:27,226 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=13, startTime=1733375846746; duration=0sec 2024-12-05T05:17:27,226 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:27,226 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:27,227 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/b9ff2d7866de4630b99b31a7c1c7b91e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b9ff2d7866de4630b99b31a7c1c7b91e 2024-12-05T05:17:27,232 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into b9ff2d7866de4630b99b31a7c1c7b91e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:27,232 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:27,232 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=13, startTime=1733375846746; duration=0sec 2024-12-05T05:17:27,232 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:27,232 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:27,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742133_1309 (size=12301) 2024-12-05T05:17:27,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T05:17:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375907423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,424 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375907423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375907423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T05:17:27,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:27,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,486 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T05:17:27,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/a38e45e40b4d4e3a9033bf9c09f79377 2024-12-05T05:17:27,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/97ce021ceb934d0f909d92cc78c2c80d is 50, key is test_row_0/C:col10/1733375846645/Put/seqid=0 2024-12-05T05:17:27,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742134_1310 (size=12301) 2024-12-05T05:17:27,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T05:17:27,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:27,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375907664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375907664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,667 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,668 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T05:17:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:27,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375907933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375907933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375907933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:27,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:27,936 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:27,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T05:17:27,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:27,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:27,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:28,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T05:17:28,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/97ce021ceb934d0f909d92cc78c2c80d 2024-12-05T05:17:28,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/47a842f6f9a8474f9799cb8c5c38160c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/47a842f6f9a8474f9799cb8c5c38160c 2024-12-05T05:17:28,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/47a842f6f9a8474f9799cb8c5c38160c, entries=150, sequenceid=322, filesize=12.0 K 2024-12-05T05:17:28,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/a38e45e40b4d4e3a9033bf9c09f79377 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/a38e45e40b4d4e3a9033bf9c09f79377 2024-12-05T05:17:28,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/a38e45e40b4d4e3a9033bf9c09f79377, entries=150, sequenceid=322, filesize=12.0 K 2024-12-05T05:17:28,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/97ce021ceb934d0f909d92cc78c2c80d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97ce021ceb934d0f909d92cc78c2c80d 2024-12-05T05:17:28,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97ce021ceb934d0f909d92cc78c2c80d, entries=150, sequenceid=322, filesize=12.0 K 2024-12-05T05:17:28,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for ac783a93e1fe6938afb704b21f8b8675 in 1321ms, sequenceid=322, compaction requested=false 2024-12-05T05:17:28,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:28,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-05T05:17:28,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:28,138 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T05:17:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:28,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:28,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/e7666e02acaa4c889f82e458843728e0 is 50, key is test_row_0/A:col10/1733375846787/Put/seqid=0 2024-12-05T05:17:28,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742135_1311 (size=12301) 2024-12-05T05:17:28,201 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/e7666e02acaa4c889f82e458843728e0 2024-12-05T05:17:28,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/fa37cc871ff44b269961e025b489113a is 50, key is test_row_0/B:col10/1733375846787/Put/seqid=0 2024-12-05T05:17:28,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742136_1312 (size=12301) 2024-12-05T05:17:28,262 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/fa37cc871ff44b269961e025b489113a 2024-12-05T05:17:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/d763521b952f4d57a28c088c667ba384 is 50, key is test_row_0/C:col10/1733375846787/Put/seqid=0 2024-12-05T05:17:28,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742137_1313 (size=12301) 2024-12-05T05:17:28,699 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/d763521b952f4d57a28c088c667ba384 2024-12-05T05:17:28,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/e7666e02acaa4c889f82e458843728e0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/e7666e02acaa4c889f82e458843728e0 2024-12-05T05:17:28,706 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/e7666e02acaa4c889f82e458843728e0, entries=150, sequenceid=339, filesize=12.0 K 2024-12-05T05:17:28,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/fa37cc871ff44b269961e025b489113a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/fa37cc871ff44b269961e025b489113a 2024-12-05T05:17:28,710 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/fa37cc871ff44b269961e025b489113a, entries=150, sequenceid=339, filesize=12.0 K 2024-12-05T05:17:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/d763521b952f4d57a28c088c667ba384 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d763521b952f4d57a28c088c667ba384 2024-12-05T05:17:28,715 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d763521b952f4d57a28c088c667ba384, entries=150, sequenceid=339, filesize=12.0 K 2024-12-05T05:17:28,715 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for ac783a93e1fe6938afb704b21f8b8675 in 577ms, sequenceid=339, compaction requested=true 2024-12-05T05:17:28,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:28,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:28,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-05T05:17:28,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-05T05:17:28,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-05T05:17:28,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8420 sec 2024-12-05T05:17:28,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.8470 sec 2024-12-05T05:17:28,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:17:28,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:28,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:28,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:28,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:28,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:28,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:28,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:28,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/2f6609fc04e4430f849bd29aa2c36929 is 50, key is test_row_0/A:col10/1733375848948/Put/seqid=0 2024-12-05T05:17:28,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742138_1314 (size=17181) 2024-12-05T05:17:28,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375908986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:28,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375848981, max timestamp=1733375848989Num uniques 859; 2024-12-05T05:17:28,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375848981, max timestamp=1733375848989Num uniques 859; 2024-12-05T05:17:28,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:28,991 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:28,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375848981, max timestamp=1733375848989Num uniques 859; 2024-12-05T05:17:28,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:29,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375909019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-05T05:17:29,023 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:29,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-05T05:17:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T05:17:29,024 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:29,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375909021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,025 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:29,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375909092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T05:17:29,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375909133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375909137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,140 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T05:17:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375909302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T05:17:29,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T05:17:29,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375909343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375909343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/2f6609fc04e4430f849bd29aa2c36929 2024-12-05T05:17:29,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c4bbc6be9cd24019a564d4834c6eb9ed is 50, key is test_row_0/B:col10/1733375848948/Put/seqid=0 2024-12-05T05:17:29,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742139_1315 (size=12301) 2024-12-05T05:17:29,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c4bbc6be9cd24019a564d4834c6eb9ed 2024-12-05T05:17:29,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/dc1630a42def4f0186a95b5d9f399f23 is 50, key is test_row_0/C:col10/1733375848948/Put/seqid=0 2024-12-05T05:17:29,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742140_1316 (size=12301) 2024-12-05T05:17:29,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T05:17:29,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:29,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375909613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,615 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T05:17:29,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:29,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T05:17:29,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375909653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375909653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375909683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,686 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375909692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:29,697 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:29,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:29,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T05:17:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:29,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/dc1630a42def4f0186a95b5d9f399f23 2024-12-05T05:17:29,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/2f6609fc04e4430f849bd29aa2c36929 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2f6609fc04e4430f849bd29aa2c36929 2024-12-05T05:17:29,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2f6609fc04e4430f849bd29aa2c36929, entries=250, sequenceid=349, filesize=16.8 K 2024-12-05T05:17:29,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/c4bbc6be9cd24019a564d4834c6eb9ed as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c4bbc6be9cd24019a564d4834c6eb9ed 2024-12-05T05:17:29,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c4bbc6be9cd24019a564d4834c6eb9ed, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T05:17:29,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/dc1630a42def4f0186a95b5d9f399f23 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/dc1630a42def4f0186a95b5d9f399f23 2024-12-05T05:17:29,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/dc1630a42def4f0186a95b5d9f399f23, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T05:17:29,814 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ac783a93e1fe6938afb704b21f8b8675 in 865ms, sequenceid=349, compaction requested=true 2024-12-05T05:17:29,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:29,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:29,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:29,814 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:29,815 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:29,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:29,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:29,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:29,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:29,816 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:29,816 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:29,816 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:29,816 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:29,816 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,816 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,816 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f3e9fc2a099a40639c43dbf903fb9b6b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/47a842f6f9a8474f9799cb8c5c38160c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/e7666e02acaa4c889f82e458843728e0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2f6609fc04e4430f849bd29aa2c36929] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=53.5 K 2024-12-05T05:17:29,816 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/0836dbddef6349589582fb4b62557a7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/a38e45e40b4d4e3a9033bf9c09f79377, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/fa37cc871ff44b269961e025b489113a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c4bbc6be9cd24019a564d4834c6eb9ed] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=48.8 K 2024-12-05T05:17:29,817 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3e9fc2a099a40639c43dbf903fb9b6b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733375845496 2024-12-05T05:17:29,817 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0836dbddef6349589582fb4b62557a7d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733375845496 2024-12-05T05:17:29,817 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47a842f6f9a8474f9799cb8c5c38160c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733375846645 2024-12-05T05:17:29,817 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a38e45e40b4d4e3a9033bf9c09f79377, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733375846645 2024-12-05T05:17:29,817 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting fa37cc871ff44b269961e025b489113a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733375846782 2024-12-05T05:17:29,817 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7666e02acaa4c889f82e458843728e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733375846782 2024-12-05T05:17:29,818 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f6609fc04e4430f849bd29aa2c36929, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375848943 2024-12-05T05:17:29,818 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c4bbc6be9cd24019a564d4834c6eb9ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375848947 2024-12-05T05:17:29,826 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:29,826 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:29,827 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/210760f051f543e487f9ab093588db1c is 50, key is test_row_0/A:col10/1733375848948/Put/seqid=0 2024-12-05T05:17:29,828 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/459f5eddf4cb4f18b5957b07975d9652 is 50, key is test_row_0/B:col10/1733375848948/Put/seqid=0 2024-12-05T05:17:29,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742141_1317 (size=13187) 2024-12-05T05:17:29,846 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/210760f051f543e487f9ab093588db1c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/210760f051f543e487f9ab093588db1c 2024-12-05T05:17:29,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742142_1318 (size=13187) 2024-12-05T05:17:29,852 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into 210760f051f543e487f9ab093588db1c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:29,852 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:29,852 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=12, startTime=1733375849814; duration=0sec 2024-12-05T05:17:29,852 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:29,852 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:29,852 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:17:29,853 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:17:29,854 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:29,854 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,854 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b9ff2d7866de4630b99b31a7c1c7b91e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97ce021ceb934d0f909d92cc78c2c80d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d763521b952f4d57a28c088c667ba384, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/dc1630a42def4f0186a95b5d9f399f23] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=48.8 K 2024-12-05T05:17:29,855 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9ff2d7866de4630b99b31a7c1c7b91e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733375845496 2024-12-05T05:17:29,855 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97ce021ceb934d0f909d92cc78c2c80d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1733375846645 2024-12-05T05:17:29,855 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/459f5eddf4cb4f18b5957b07975d9652 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/459f5eddf4cb4f18b5957b07975d9652 2024-12-05T05:17:29,855 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting d763521b952f4d57a28c088c667ba384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=339, earliestPutTs=1733375846782 2024-12-05T05:17:29,856 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc1630a42def4f0186a95b5d9f399f23, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375848947 2024-12-05T05:17:29,860 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 459f5eddf4cb4f18b5957b07975d9652(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:29,860 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:29,860 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=12, startTime=1733375849814; duration=0sec 2024-12-05T05:17:29,860 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:29,860 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:29,864 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#272 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:29,864 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/acffea6edb474a88ad7b1ea402023105 is 50, key is test_row_0/C:col10/1733375848948/Put/seqid=0 2024-12-05T05:17:29,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742143_1319 (size=13187) 2024-12-05T05:17:29,873 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/acffea6edb474a88ad7b1ea402023105 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/acffea6edb474a88ad7b1ea402023105 2024-12-05T05:17:29,878 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into acffea6edb474a88ad7b1ea402023105(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:29,878 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:29,878 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=12, startTime=1733375849815; duration=0sec 2024-12-05T05:17:29,878 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:29,878 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:29,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-05T05:17:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:29,943 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T05:17:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:29,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:29,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:29,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/b061c95278be4138aeed592258a5729b is 50, key is test_row_0/A:col10/1733375849020/Put/seqid=0 2024-12-05T05:17:29,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742144_1320 (size=12301) 2024-12-05T05:17:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:30,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. as already flushing 2024-12-05T05:17:30,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375910151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T05:17:30,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:30,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375910169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:30,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375910169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:30,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:30,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375910262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:30,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:30,358 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/b061c95278be4138aeed592258a5729b 2024-12-05T05:17:30,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/d0c93a746d7547839bd49a685d85d4be is 50, key is test_row_0/B:col10/1733375849020/Put/seqid=0 2024-12-05T05:17:30,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742145_1321 (size=12301) 2024-12-05T05:17:30,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375910473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:30,770 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/d0c93a746d7547839bd49a685d85d4be 2024-12-05T05:17:30,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ea838c4bfdf64724b3aa07c0d1073743 is 50, key is test_row_0/C:col10/1733375849020/Put/seqid=0 2024-12-05T05:17:30,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742146_1322 (size=12301) 2024-12-05T05:17:30,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:30,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375910782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:30,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:31,180 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ea838c4bfdf64724b3aa07c0d1073743 2024-12-05T05:17:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T05:17:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375911182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:31,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:31,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/b061c95278be4138aeed592258a5729b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/b061c95278be4138aeed592258a5729b 2024-12-05T05:17:31,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375911183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:31,185 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:31,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:31,190 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:31,193 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/b061c95278be4138aeed592258a5729b, entries=150, sequenceid=389, filesize=12.0 K 2024-12-05T05:17:31,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/d0c93a746d7547839bd49a685d85d4be as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d0c93a746d7547839bd49a685d85d4be 2024-12-05T05:17:31,197 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d0c93a746d7547839bd49a685d85d4be, entries=150, sequenceid=389, filesize=12.0 K 2024-12-05T05:17:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/ea838c4bfdf64724b3aa07c0d1073743 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ea838c4bfdf64724b3aa07c0d1073743 2024-12-05T05:17:31,202 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ea838c4bfdf64724b3aa07c0d1073743, entries=150, sequenceid=389, filesize=12.0 K 2024-12-05T05:17:31,203 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ac783a93e1fe6938afb704b21f8b8675 in 1260ms, sequenceid=389, compaction requested=false 2024-12-05T05:17:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:31,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:31,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-05T05:17:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-05T05:17:31,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-05T05:17:31,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-12-05T05:17:31,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.1840 sec 2024-12-05T05:17:31,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:31,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:17:31,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:31,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:31,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:31,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:31,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:31,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:31,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/7b8c6e4001b249789d583d9df39ff453 is 50, key is test_row_0/A:col10/1733375850144/Put/seqid=0 2024-12-05T05:17:31,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742147_1323 (size=14741) 2024-12-05T05:17:31,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375851341, max timestamp=1733375851366Num uniques 859; 2024-12-05T05:17:31,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375851341, max timestamp=1733375851366Num uniques 859; 2024-12-05T05:17:31,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:31,368 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:31,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375851341, max timestamp=1733375851366Num uniques 859; 2024-12-05T05:17:31,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:31,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375911449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:31,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:31,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:31,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375911562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:31,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/7b8c6e4001b249789d583d9df39ff453 2024-12-05T05:17:31,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/88f694084c324c0e917c07cf0f5a1b01 is 50, key is test_row_0/B:col10/1733375850144/Put/seqid=0 2024-12-05T05:17:31,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742148_1324 (size=12301) 2024-12-05T05:17:31,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375911773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:32,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:32,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375912082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:32,085 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:32,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:32,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/88f694084c324c0e917c07cf0f5a1b01 2024-12-05T05:17:32,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/b59c501d12344c328a487881fa99fb21 is 50, key is test_row_0/C:col10/1733375850144/Put/seqid=0 2024-12-05T05:17:32,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742149_1325 (size=12301) 2024-12-05T05:17:32,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/b59c501d12344c328a487881fa99fb21 2024-12-05T05:17:32,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/7b8c6e4001b249789d583d9df39ff453 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7b8c6e4001b249789d583d9df39ff453 2024-12-05T05:17:32,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7b8c6e4001b249789d583d9df39ff453, entries=200, sequenceid=400, filesize=14.4 K 2024-12-05T05:17:32,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/88f694084c324c0e917c07cf0f5a1b01 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/88f694084c324c0e917c07cf0f5a1b01 2024-12-05T05:17:32,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/88f694084c324c0e917c07cf0f5a1b01, entries=150, sequenceid=400, filesize=12.0 K 2024-12-05T05:17:32,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/b59c501d12344c328a487881fa99fb21 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b59c501d12344c328a487881fa99fb21 2024-12-05T05:17:32,548 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b59c501d12344c328a487881fa99fb21, entries=150, sequenceid=400, filesize=12.0 K 2024-12-05T05:17:32,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ac783a93e1fe6938afb704b21f8b8675 in 1254ms, sequenceid=400, compaction requested=true 2024-12-05T05:17:32,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:32,550 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:32,550 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:32,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:32,550 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:32,550 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:32,550 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:32,550 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:32,551 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:32,551 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:32,551 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/210760f051f543e487f9ab093588db1c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/b061c95278be4138aeed592258a5729b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7b8c6e4001b249789d583d9df39ff453] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=39.3 K 2024-12-05T05:17:32,551 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/459f5eddf4cb4f18b5957b07975d9652, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d0c93a746d7547839bd49a685d85d4be, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/88f694084c324c0e917c07cf0f5a1b01] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.9 K 2024-12-05T05:17:32,551 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 210760f051f543e487f9ab093588db1c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375848947 2024-12-05T05:17:32,551 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 459f5eddf4cb4f18b5957b07975d9652, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375848947 2024-12-05T05:17:32,551 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c93a746d7547839bd49a685d85d4be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733375849009 2024-12-05T05:17:32,551 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b061c95278be4138aeed592258a5729b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733375849009 2024-12-05T05:17:32,552 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 88f694084c324c0e917c07cf0f5a1b01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375850140 2024-12-05T05:17:32,552 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b8c6e4001b249789d583d9df39ff453, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375850140 2024-12-05T05:17:32,559 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:32,560 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/33249c8a42d3461ea752edec259114f5 is 50, key is test_row_0/B:col10/1733375850144/Put/seqid=0 2024-12-05T05:17:32,572 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:32,572 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f35174674e7e4e679f0af396046d8d73 is 50, key is test_row_0/A:col10/1733375850144/Put/seqid=0 2024-12-05T05:17:32,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742150_1326 (size=13289) 2024-12-05T05:17:32,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742151_1327 (size=13289) 2024-12-05T05:17:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:32,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T05:17:32,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:32,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:32,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:32,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:32,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:32,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:32,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/c9f810b4ea4642cfa71a3d4918c36a0b is 50, key is test_row_0/A:col10/1733375852595/Put/seqid=0 2024-12-05T05:17:32,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742152_1328 (size=14741) 2024-12-05T05:17:32,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375912630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:32,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:32,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375912742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:32,818 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-12-05T05:17:32,818 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:32,818 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:32,818 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,819 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-12-05T05:17:32,819 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:32,819 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,819 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,819 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-12-05T05:17:32,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:32,819 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:32,819 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:32,819 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,822 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-12-05T05:17:32,822 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:32,822 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,822 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:32,823 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-12-05T05:17:32,823 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:32,823 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,823 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:32,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:32,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375912953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:32,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:32,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:32,986 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/33249c8a42d3461ea752edec259114f5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/33249c8a42d3461ea752edec259114f5 2024-12-05T05:17:32,986 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/f35174674e7e4e679f0af396046d8d73 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f35174674e7e4e679f0af396046d8d73 2024-12-05T05:17:32,992 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into f35174674e7e4e679f0af396046d8d73(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:32,992 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 33249c8a42d3461ea752edec259114f5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:32,992 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:32,992 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:32,993 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=13, startTime=1733375852550; duration=0sec 2024-12-05T05:17:32,993 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=13, startTime=1733375852549; duration=0sec 2024-12-05T05:17:32,993 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:32,993 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:32,993 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:32,993 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:32,993 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:32,994 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:32,994 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:32,994 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:32,994 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/acffea6edb474a88ad7b1ea402023105, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ea838c4bfdf64724b3aa07c0d1073743, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b59c501d12344c328a487881fa99fb21] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=36.9 K 2024-12-05T05:17:32,994 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting acffea6edb474a88ad7b1ea402023105, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375848947 2024-12-05T05:17:32,994 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ea838c4bfdf64724b3aa07c0d1073743, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1733375849009 2024-12-05T05:17:32,995 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b59c501d12344c328a487881fa99fb21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375850140 2024-12-05T05:17:33,000 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:33,000 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/113e91d77a91414fa5ee7c1a79883259 is 50, key is test_row_0/C:col10/1733375850144/Put/seqid=0 2024-12-05T05:17:33,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742153_1329 (size=13289) 2024-12-05T05:17:33,004 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/c9f810b4ea4642cfa71a3d4918c36a0b 2024-12-05T05:17:33,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/6b0670a5bfe64aa5ba2fd715d52f4a31 is 50, key is test_row_0/B:col10/1733375852595/Put/seqid=0 2024-12-05T05:17:33,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742154_1330 (size=12301) 2024-12-05T05:17:33,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:33,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50204 deadline: 1733375913192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:33,193 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-05T05:17:33,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:33,203 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50270 deadline: 1733375913203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:33,204 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:33,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375913263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:33,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/6b0670a5bfe64aa5ba2fd715d52f4a31 2024-12-05T05:17:33,416 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/113e91d77a91414fa5ee7c1a79883259 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/113e91d77a91414fa5ee7c1a79883259 2024-12-05T05:17:33,422 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into 113e91d77a91414fa5ee7c1a79883259(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:33,422 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:33,422 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=13, startTime=1733375852550; duration=0sec 2024-12-05T05:17:33,422 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:33,422 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:33,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/62fbf3242f3143688d5d9775ee1189c6 is 50, key is test_row_0/C:col10/1733375852595/Put/seqid=0 2024-12-05T05:17:33,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742155_1331 (size=12301) 2024-12-05T05:17:33,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50284 deadline: 1733375913703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:33,706 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:33,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:33,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50238 deadline: 1733375913723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:33,724 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ac783a93e1fe6938afb704b21f8b8675 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:33,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:50296 deadline: 1733375913773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:33,773 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:17:33,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/62fbf3242f3143688d5d9775ee1189c6 2024-12-05T05:17:33,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/c9f810b4ea4642cfa71a3d4918c36a0b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/c9f810b4ea4642cfa71a3d4918c36a0b 2024-12-05T05:17:33,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/c9f810b4ea4642cfa71a3d4918c36a0b, entries=200, sequenceid=437, filesize=14.4 K 2024-12-05T05:17:33,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/6b0670a5bfe64aa5ba2fd715d52f4a31 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6b0670a5bfe64aa5ba2fd715d52f4a31 2024-12-05T05:17:33,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6b0670a5bfe64aa5ba2fd715d52f4a31, entries=150, sequenceid=437, filesize=12.0 K 2024-12-05T05:17:33,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/62fbf3242f3143688d5d9775ee1189c6 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/62fbf3242f3143688d5d9775ee1189c6 2024-12-05T05:17:33,843 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/62fbf3242f3143688d5d9775ee1189c6, entries=150, sequenceid=437, filesize=12.0 K 2024-12-05T05:17:33,843 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ac783a93e1fe6938afb704b21f8b8675 in 1247ms, sequenceid=437, compaction requested=false 2024-12-05T05:17:33,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:34,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:17:34,784 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-12-05T05:17:34,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:34,784 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:34,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:34,784 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:34,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:34,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:34,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:34,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:34,784 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:34,784 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:34,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/2d7f47a9e213429fb4c970aed6cb4432 is 50, key is test_row_0/A:col10/1733375854783/Put/seqid=0 2024-12-05T05:17:34,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742156_1332 (size=9857) 2024-12-05T05:17:35,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/2d7f47a9e213429fb4c970aed6cb4432 2024-12-05T05:17:35,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/938c9d20edf54eb38e3cc266a1d68cac is 50, key is test_row_0/B:col10/1733375854783/Put/seqid=0 2024-12-05T05:17:35,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742157_1333 (size=9857) 2024-12-05T05:17:35,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/938c9d20edf54eb38e3cc266a1d68cac 2024-12-05T05:17:35,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/eee04919226841b1b06edaef380d4d20 is 50, key is test_row_0/C:col10/1733375854783/Put/seqid=0 2024-12-05T05:17:35,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742158_1334 (size=9857) 2024-12-05T05:17:36,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/eee04919226841b1b06edaef380d4d20 2024-12-05T05:17:36,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/2d7f47a9e213429fb4c970aed6cb4432 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2d7f47a9e213429fb4c970aed6cb4432 2024-12-05T05:17:36,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2d7f47a9e213429fb4c970aed6cb4432, entries=100, sequenceid=451, filesize=9.6 K 2024-12-05T05:17:36,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/938c9d20edf54eb38e3cc266a1d68cac as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/938c9d20edf54eb38e3cc266a1d68cac 2024-12-05T05:17:36,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/938c9d20edf54eb38e3cc266a1d68cac, entries=100, sequenceid=451, filesize=9.6 K 2024-12-05T05:17:36,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/eee04919226841b1b06edaef380d4d20 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/eee04919226841b1b06edaef380d4d20 2024-12-05T05:17:36,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/eee04919226841b1b06edaef380d4d20, entries=100, sequenceid=451, filesize=9.6 K 2024-12-05T05:17:36,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for ac783a93e1fe6938afb704b21f8b8675 in 1259ms, sequenceid=451, compaction requested=true 2024-12-05T05:17:36,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:36,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:36,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:36,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:36,044 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:36,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:36,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac783a93e1fe6938afb704b21f8b8675:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:36,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:36,044 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37887 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/B is initiating minor compaction (all files) 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/A is initiating minor compaction (all files) 2024-12-05T05:17:36,045 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/B in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:36,045 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/A in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:36,045 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/33249c8a42d3461ea752edec259114f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6b0670a5bfe64aa5ba2fd715d52f4a31, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/938c9d20edf54eb38e3cc266a1d68cac] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=34.6 K 2024-12-05T05:17:36,045 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f35174674e7e4e679f0af396046d8d73, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/c9f810b4ea4642cfa71a3d4918c36a0b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2d7f47a9e213429fb4c970aed6cb4432] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=37.0 K 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f35174674e7e4e679f0af396046d8d73, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375850140 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 33249c8a42d3461ea752edec259114f5, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375850140 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0670a5bfe64aa5ba2fd715d52f4a31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733375851431 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9f810b4ea4642cfa71a3d4918c36a0b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733375851431 2024-12-05T05:17:36,045 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 938c9d20edf54eb38e3cc266a1d68cac, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733375852615 2024-12-05T05:17:36,046 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d7f47a9e213429fb4c970aed6cb4432, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733375852615 2024-12-05T05:17:36,053 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#A#compaction#288 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:36,053 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#B#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:36,053 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/7f6d3cfff79644da8b0b698a989c30a3 is 50, key is test_row_0/B:col10/1733375854783/Put/seqid=0 2024-12-05T05:17:36,053 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/ead6ed85bafe4d338c207f33103807c7 is 50, key is test_row_0/A:col10/1733375854783/Put/seqid=0 2024-12-05T05:17:36,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742160_1336 (size=13391) 2024-12-05T05:17:36,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742159_1335 (size=13391) 2024-12-05T05:17:36,469 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/7f6d3cfff79644da8b0b698a989c30a3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7f6d3cfff79644da8b0b698a989c30a3 2024-12-05T05:17:36,470 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/ead6ed85bafe4d338c207f33103807c7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ead6ed85bafe4d338c207f33103807c7 2024-12-05T05:17:36,474 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/B of ac783a93e1fe6938afb704b21f8b8675 into 7f6d3cfff79644da8b0b698a989c30a3(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:36,474 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:36,474 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/B, priority=13, startTime=1733375856044; duration=0sec 2024-12-05T05:17:36,474 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:36,474 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:B 2024-12-05T05:17:36,474 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:36,474 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/A of ac783a93e1fe6938afb704b21f8b8675 into ead6ed85bafe4d338c207f33103807c7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:36,474 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:36,475 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/A, priority=13, startTime=1733375856044; duration=0sec 2024-12-05T05:17:36,475 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:36,475 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:A 2024-12-05T05:17:36,475 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:36,475 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ac783a93e1fe6938afb704b21f8b8675/C is initiating minor compaction (all files) 2024-12-05T05:17:36,475 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac783a93e1fe6938afb704b21f8b8675/C in TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:36,475 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/113e91d77a91414fa5ee7c1a79883259, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/62fbf3242f3143688d5d9775ee1189c6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/eee04919226841b1b06edaef380d4d20] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp, totalSize=34.6 K 2024-12-05T05:17:36,475 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 113e91d77a91414fa5ee7c1a79883259, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1733375850140 2024-12-05T05:17:36,476 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fbf3242f3143688d5d9775ee1189c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1733375851431 2024-12-05T05:17:36,476 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting eee04919226841b1b06edaef380d4d20, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733375852615 2024-12-05T05:17:36,483 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac783a93e1fe6938afb704b21f8b8675#C#compaction#290 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:36,484 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/fa31d32c630e4ee0ab57af1d11ba5da5 is 50, key is test_row_0/C:col10/1733375854783/Put/seqid=0 2024-12-05T05:17:36,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742161_1337 (size=13391) 2024-12-05T05:17:36,894 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/fa31d32c630e4ee0ab57af1d11ba5da5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/fa31d32c630e4ee0ab57af1d11ba5da5 2024-12-05T05:17:36,899 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac783a93e1fe6938afb704b21f8b8675/C of ac783a93e1fe6938afb704b21f8b8675 into fa31d32c630e4ee0ab57af1d11ba5da5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:36,899 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac783a93e1fe6938afb704b21f8b8675: 2024-12-05T05:17:36,899 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., storeName=ac783a93e1fe6938afb704b21f8b8675/C, priority=13, startTime=1733375856044; duration=0sec 2024-12-05T05:17:36,899 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:36,899 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac783a93e1fe6938afb704b21f8b8675:C 2024-12-05T05:17:37,225 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-12-05T05:17:37,225 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:37,225 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:37,226 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:37,226 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:37,236 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-12-05T05:17:37,236 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:37,236 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:37,236 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:37,236 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:39,518 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T05:17:43,785 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-12-05T05:17:43,785 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:43,785 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:43,785 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:43,785 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:43,815 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-12-05T05:17:43,815 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:17:43,815 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:43,815 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:43,815 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 68 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4076 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3860 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4042 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4069 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3860 2024-12-05T05:17:43,815 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T05:17:43,816 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T05:17:43,816 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:17:43,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:43,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:43,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:17:43,817 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T05:17:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:43,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T05:17:43,820 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375863820"}]},"ts":"1733375863820"} 2024-12-05T05:17:43,822 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T05:17:43,822 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T05:17:43,822 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:17:43,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, UNASSIGN}] 2024-12-05T05:17:43,825 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, UNASSIGN 2024-12-05T05:17:43,825 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=ac783a93e1fe6938afb704b21f8b8675, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:43,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, UNASSIGN because future has completed 2024-12-05T05:17:43,826 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:17:43,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:17:43,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T05:17:43,979 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:43,979 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:17:43,979 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing ac783a93e1fe6938afb704b21f8b8675, disabling compactions & flushes 2024-12-05T05:17:43,979 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:43,979 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:43,979 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. after waiting 0 ms 2024-12-05T05:17:43,979 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:43,979 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(2902): Flushing ac783a93e1fe6938afb704b21f8b8675 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T05:17:43,980 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=A 2024-12-05T05:17:43,980 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:43,980 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=B 2024-12-05T05:17:43,980 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:43,980 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac783a93e1fe6938afb704b21f8b8675, store=C 2024-12-05T05:17:43,980 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:43,984 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/58af783f680346d0846db2a0efa1df00 is 50, key is test_row_1/A:col10/1733375863813/Put/seqid=0 2024-12-05T05:17:43,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742162_1338 (size=9857) 2024-12-05T05:17:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T05:17:44,389 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/58af783f680346d0846db2a0efa1df00 2024-12-05T05:17:44,398 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/86719054505f431282784ca5e70ab078 is 50, key is test_row_1/B:col10/1733375863813/Put/seqid=0 2024-12-05T05:17:44,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742163_1339 (size=9857) 2024-12-05T05:17:44,406 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/86719054505f431282784ca5e70ab078 2024-12-05T05:17:44,417 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/f922aca1cd3e45c2a37fa4c167e594a4 is 50, key is test_row_1/C:col10/1733375863813/Put/seqid=0 2024-12-05T05:17:44,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742164_1340 (size=9857) 2024-12-05T05:17:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T05:17:44,825 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=461 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/f922aca1cd3e45c2a37fa4c167e594a4 2024-12-05T05:17:44,830 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/A/58af783f680346d0846db2a0efa1df00 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/58af783f680346d0846db2a0efa1df00 2024-12-05T05:17:44,837 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/58af783f680346d0846db2a0efa1df00, entries=100, sequenceid=461, filesize=9.6 K 2024-12-05T05:17:44,838 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/B/86719054505f431282784ca5e70ab078 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/86719054505f431282784ca5e70ab078 2024-12-05T05:17:44,841 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/86719054505f431282784ca5e70ab078, entries=100, sequenceid=461, filesize=9.6 K 2024-12-05T05:17:44,842 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/.tmp/C/f922aca1cd3e45c2a37fa4c167e594a4 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f922aca1cd3e45c2a37fa4c167e594a4 2024-12-05T05:17:44,846 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f922aca1cd3e45c2a37fa4c167e594a4, entries=100, sequenceid=461, filesize=9.6 K 2024-12-05T05:17:44,847 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ac783a93e1fe6938afb704b21f8b8675 in 868ms, sequenceid=461, compaction requested=false 2024-12-05T05:17:44,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5ac76b7c712f4108b1b9d7f15ebbae78, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6497a1b420e04f9ea068dd9bfe21a35c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/135803f8dcef470485b0adfb51ea88c6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/313136bebb3a4b37832e8b9cc8a7e9e1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/fbe6715473894acd88da9865ee84ae40, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6b6ad38f6fa24dc2908aa13893676564, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d6140543c9dd498896c50f1fed975f51, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/edcedefb69834df3868052970f6ddb69, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/035a9f300c1d49d390fe78e42b9ec610, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/cbca17cd7a0b416487f782c522df7aca, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d997c3a6ff4f42ffabe9727d38bf926e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ef44aa40f8f640d8bda863f2f02147a0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/4e0b4c6a184d43128f4b6f901dcacf97, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5331bf6c296c4bc196c6ec176250752b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/0325d7f41da14ff591ac8fe1c7ae6268, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f585fb3780e543239788d8bc9299d2f3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/9a2c8ee9e8c945fa9071361cc285d7ab, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f01fb7b0a1a143729eb9236558a41320, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/aef3449dfb744a92a9f8cddff493946b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7f96737bcb1a4d7ea32334e594016d3e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/700c927d77b64b8bac949193c3b7b557, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/a683775a1f854132820a0b6609bf0272, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f3e9fc2a099a40639c43dbf903fb9b6b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/47a842f6f9a8474f9799cb8c5c38160c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/e7666e02acaa4c889f82e458843728e0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2f6609fc04e4430f849bd29aa2c36929, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/210760f051f543e487f9ab093588db1c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/b061c95278be4138aeed592258a5729b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7b8c6e4001b249789d583d9df39ff453, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f35174674e7e4e679f0af396046d8d73, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/c9f810b4ea4642cfa71a3d4918c36a0b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2d7f47a9e213429fb4c970aed6cb4432] to archive 2024-12-05T05:17:44,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:17:44,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5ac76b7c712f4108b1b9d7f15ebbae78 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5ac76b7c712f4108b1b9d7f15ebbae78 2024-12-05T05:17:44,851 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6497a1b420e04f9ea068dd9bfe21a35c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6497a1b420e04f9ea068dd9bfe21a35c 2024-12-05T05:17:44,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/135803f8dcef470485b0adfb51ea88c6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/135803f8dcef470485b0adfb51ea88c6 2024-12-05T05:17:44,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/313136bebb3a4b37832e8b9cc8a7e9e1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/313136bebb3a4b37832e8b9cc8a7e9e1 2024-12-05T05:17:44,854 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/fbe6715473894acd88da9865ee84ae40 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/fbe6715473894acd88da9865ee84ae40 2024-12-05T05:17:44,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6b6ad38f6fa24dc2908aa13893676564 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/6b6ad38f6fa24dc2908aa13893676564 2024-12-05T05:17:44,856 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d6140543c9dd498896c50f1fed975f51 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d6140543c9dd498896c50f1fed975f51 2024-12-05T05:17:44,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/edcedefb69834df3868052970f6ddb69 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/edcedefb69834df3868052970f6ddb69 2024-12-05T05:17:44,858 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/035a9f300c1d49d390fe78e42b9ec610 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/035a9f300c1d49d390fe78e42b9ec610 2024-12-05T05:17:44,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/cbca17cd7a0b416487f782c522df7aca to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/cbca17cd7a0b416487f782c522df7aca 2024-12-05T05:17:44,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d997c3a6ff4f42ffabe9727d38bf926e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/d997c3a6ff4f42ffabe9727d38bf926e 2024-12-05T05:17:44,861 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ef44aa40f8f640d8bda863f2f02147a0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ef44aa40f8f640d8bda863f2f02147a0 2024-12-05T05:17:44,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/4e0b4c6a184d43128f4b6f901dcacf97 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/4e0b4c6a184d43128f4b6f901dcacf97 2024-12-05T05:17:44,863 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5331bf6c296c4bc196c6ec176250752b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/5331bf6c296c4bc196c6ec176250752b 2024-12-05T05:17:44,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/0325d7f41da14ff591ac8fe1c7ae6268 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/0325d7f41da14ff591ac8fe1c7ae6268 2024-12-05T05:17:44,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f585fb3780e543239788d8bc9299d2f3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f585fb3780e543239788d8bc9299d2f3 2024-12-05T05:17:44,865 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/9a2c8ee9e8c945fa9071361cc285d7ab to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/9a2c8ee9e8c945fa9071361cc285d7ab 2024-12-05T05:17:44,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f01fb7b0a1a143729eb9236558a41320 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f01fb7b0a1a143729eb9236558a41320 2024-12-05T05:17:44,867 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/aef3449dfb744a92a9f8cddff493946b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/aef3449dfb744a92a9f8cddff493946b 2024-12-05T05:17:44,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7f96737bcb1a4d7ea32334e594016d3e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7f96737bcb1a4d7ea32334e594016d3e 2024-12-05T05:17:44,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/700c927d77b64b8bac949193c3b7b557 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/700c927d77b64b8bac949193c3b7b557 2024-12-05T05:17:44,872 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/a683775a1f854132820a0b6609bf0272 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/a683775a1f854132820a0b6609bf0272 2024-12-05T05:17:44,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f3e9fc2a099a40639c43dbf903fb9b6b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f3e9fc2a099a40639c43dbf903fb9b6b 2024-12-05T05:17:44,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/47a842f6f9a8474f9799cb8c5c38160c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/47a842f6f9a8474f9799cb8c5c38160c 2024-12-05T05:17:44,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/e7666e02acaa4c889f82e458843728e0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/e7666e02acaa4c889f82e458843728e0 2024-12-05T05:17:44,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2f6609fc04e4430f849bd29aa2c36929 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2f6609fc04e4430f849bd29aa2c36929 2024-12-05T05:17:44,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/210760f051f543e487f9ab093588db1c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/210760f051f543e487f9ab093588db1c 2024-12-05T05:17:44,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/b061c95278be4138aeed592258a5729b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/b061c95278be4138aeed592258a5729b 2024-12-05T05:17:44,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7b8c6e4001b249789d583d9df39ff453 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/7b8c6e4001b249789d583d9df39ff453 2024-12-05T05:17:44,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f35174674e7e4e679f0af396046d8d73 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/f35174674e7e4e679f0af396046d8d73 2024-12-05T05:17:44,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/c9f810b4ea4642cfa71a3d4918c36a0b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/c9f810b4ea4642cfa71a3d4918c36a0b 2024-12-05T05:17:44,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2d7f47a9e213429fb4c970aed6cb4432 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/2d7f47a9e213429fb4c970aed6cb4432 2024-12-05T05:17:44,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/63b5f150a1e5467693cfe5cd5306c5b1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/66d2c8f726b54296a0f45f516dddc80c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/70265c04bce544dc8bfdfcf1788840ca, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/06d4c9c77ab04472bbeb450cb2744e7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8e0c2e93e62b480f8481b1a556414098, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c53c0b7cb855419dbeec9bec408659ba, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/28f26a7c7894437ba7ba290eaa8964fc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da955ca184d548db92da97be104f63d5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9bb787550e2418faa107deb5f777e10, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da7fc6d2e3c043b2b095fbc7d3137b3e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/1052c79f55444d66b3a11b139faef833, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/40d5b21e7ad74826b7d3e913461b3e9b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8286231e4fa543dcb95867e3ddd6cc3d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d5fe6b2b0e5d4dba9947f0a9c12f1722, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6332144b00814784a4c6d1b4e54af53d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/303fbc2aca1b4e12bdf3aa677e59ecaf, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/08ea1ec8dd6f45cb8aee547ebbae97d2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/04d4bff0e71c4472a5616a4304c3601e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6335907b59e94128ab8dd8341ec5ee3d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7d5426a2cbb0422895dc4551133df550, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/16605c3466d74de1bb9f36805990ac3e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/0836dbddef6349589582fb4b62557a7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9853172ecaf4c3bb3d23d8641505138, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/a38e45e40b4d4e3a9033bf9c09f79377, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/fa37cc871ff44b269961e025b489113a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/459f5eddf4cb4f18b5957b07975d9652, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c4bbc6be9cd24019a564d4834c6eb9ed, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d0c93a746d7547839bd49a685d85d4be, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/33249c8a42d3461ea752edec259114f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/88f694084c324c0e917c07cf0f5a1b01, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6b0670a5bfe64aa5ba2fd715d52f4a31, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/938c9d20edf54eb38e3cc266a1d68cac] to archive 2024-12-05T05:17:44,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:17:44,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/63b5f150a1e5467693cfe5cd5306c5b1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/63b5f150a1e5467693cfe5cd5306c5b1 2024-12-05T05:17:44,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/66d2c8f726b54296a0f45f516dddc80c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/66d2c8f726b54296a0f45f516dddc80c 2024-12-05T05:17:44,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/70265c04bce544dc8bfdfcf1788840ca to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/70265c04bce544dc8bfdfcf1788840ca 2024-12-05T05:17:44,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/06d4c9c77ab04472bbeb450cb2744e7d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/06d4c9c77ab04472bbeb450cb2744e7d 2024-12-05T05:17:44,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8e0c2e93e62b480f8481b1a556414098 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8e0c2e93e62b480f8481b1a556414098 2024-12-05T05:17:44,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c53c0b7cb855419dbeec9bec408659ba to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c53c0b7cb855419dbeec9bec408659ba 2024-12-05T05:17:44,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/28f26a7c7894437ba7ba290eaa8964fc to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/28f26a7c7894437ba7ba290eaa8964fc 2024-12-05T05:17:44,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da955ca184d548db92da97be104f63d5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da955ca184d548db92da97be104f63d5 2024-12-05T05:17:44,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9bb787550e2418faa107deb5f777e10 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9bb787550e2418faa107deb5f777e10 2024-12-05T05:17:44,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da7fc6d2e3c043b2b095fbc7d3137b3e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/da7fc6d2e3c043b2b095fbc7d3137b3e 2024-12-05T05:17:44,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/1052c79f55444d66b3a11b139faef833 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/1052c79f55444d66b3a11b139faef833 2024-12-05T05:17:44,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/40d5b21e7ad74826b7d3e913461b3e9b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/40d5b21e7ad74826b7d3e913461b3e9b 2024-12-05T05:17:44,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8286231e4fa543dcb95867e3ddd6cc3d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/8286231e4fa543dcb95867e3ddd6cc3d 2024-12-05T05:17:44,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d5fe6b2b0e5d4dba9947f0a9c12f1722 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d5fe6b2b0e5d4dba9947f0a9c12f1722 2024-12-05T05:17:44,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6332144b00814784a4c6d1b4e54af53d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6332144b00814784a4c6d1b4e54af53d 2024-12-05T05:17:44,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/303fbc2aca1b4e12bdf3aa677e59ecaf to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/303fbc2aca1b4e12bdf3aa677e59ecaf 2024-12-05T05:17:44,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/08ea1ec8dd6f45cb8aee547ebbae97d2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/08ea1ec8dd6f45cb8aee547ebbae97d2 2024-12-05T05:17:44,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/04d4bff0e71c4472a5616a4304c3601e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/04d4bff0e71c4472a5616a4304c3601e 2024-12-05T05:17:44,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6335907b59e94128ab8dd8341ec5ee3d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6335907b59e94128ab8dd8341ec5ee3d 2024-12-05T05:17:44,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7d5426a2cbb0422895dc4551133df550 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7d5426a2cbb0422895dc4551133df550 2024-12-05T05:17:44,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/16605c3466d74de1bb9f36805990ac3e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/16605c3466d74de1bb9f36805990ac3e 2024-12-05T05:17:44,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/0836dbddef6349589582fb4b62557a7d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/0836dbddef6349589582fb4b62557a7d 2024-12-05T05:17:44,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9853172ecaf4c3bb3d23d8641505138 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c9853172ecaf4c3bb3d23d8641505138 2024-12-05T05:17:44,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/a38e45e40b4d4e3a9033bf9c09f79377 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/a38e45e40b4d4e3a9033bf9c09f79377 2024-12-05T05:17:44,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/fa37cc871ff44b269961e025b489113a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/fa37cc871ff44b269961e025b489113a 2024-12-05T05:17:44,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/459f5eddf4cb4f18b5957b07975d9652 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/459f5eddf4cb4f18b5957b07975d9652 2024-12-05T05:17:44,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c4bbc6be9cd24019a564d4834c6eb9ed to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/c4bbc6be9cd24019a564d4834c6eb9ed 2024-12-05T05:17:44,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d0c93a746d7547839bd49a685d85d4be to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/d0c93a746d7547839bd49a685d85d4be 2024-12-05T05:17:44,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/33249c8a42d3461ea752edec259114f5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/33249c8a42d3461ea752edec259114f5 2024-12-05T05:17:44,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/88f694084c324c0e917c07cf0f5a1b01 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/88f694084c324c0e917c07cf0f5a1b01 2024-12-05T05:17:44,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6b0670a5bfe64aa5ba2fd715d52f4a31 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/6b0670a5bfe64aa5ba2fd715d52f4a31 2024-12-05T05:17:44,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/938c9d20edf54eb38e3cc266a1d68cac to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/938c9d20edf54eb38e3cc266a1d68cac 2024-12-05T05:17:44,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f4a51583b6524a31a8a96ba446f0895e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/7270eb9f183441f89739d422610cd53c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/42621997bed64497a22994f79c093dd0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ac6a63dc3add4c828224a41a9cf9cf89, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d86a9ecd60a645f2a7b9e8f55fae5546, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e31e47e8efea4a6b9d218ea35c25924d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97bc242f5c314ae7b43a1700cbe09631, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/9332e0e1302045fcb518885951706551, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/059aba8cab964060a64a1c7cc75f8cce, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/1952548d9bb34aada1afe0bb629011fb, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e37f20149638410bb47e9782fff09b52, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ebd9df7432a64ed68dab791d34ed26ae, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c6d93bd62ec54cbe8b9fe149e921d847, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/58f4c2124cda42088ab8be8860da1184, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c4ac8790c0d542dbbb99b2d78bea7309, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/6a707db7706a42aa89f01564345fe355, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e54f7c54c86944e099e7f6f0f2ccbe12, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/388da213f1874d2190afe3317598c09b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/842a7ebd3a6d4696bfa6432ae4813b0a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e55f44cc77994a6e959d9cbc2743fc9f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/10f5fea1d67f46c19d7cf3eecd0fb80f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b9ff2d7866de4630b99b31a7c1c7b91e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/38b22618d56d417cb213c73f2986c4f1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97ce021ceb934d0f909d92cc78c2c80d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d763521b952f4d57a28c088c667ba384, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/acffea6edb474a88ad7b1ea402023105, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/dc1630a42def4f0186a95b5d9f399f23, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ea838c4bfdf64724b3aa07c0d1073743, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/113e91d77a91414fa5ee7c1a79883259, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b59c501d12344c328a487881fa99fb21, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/62fbf3242f3143688d5d9775ee1189c6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/eee04919226841b1b06edaef380d4d20] to archive 2024-12-05T05:17:44,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:17:44,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f4a51583b6524a31a8a96ba446f0895e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f4a51583b6524a31a8a96ba446f0895e 2024-12-05T05:17:44,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/7270eb9f183441f89739d422610cd53c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/7270eb9f183441f89739d422610cd53c 2024-12-05T05:17:44,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/42621997bed64497a22994f79c093dd0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/42621997bed64497a22994f79c093dd0 2024-12-05T05:17:44,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ac6a63dc3add4c828224a41a9cf9cf89 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ac6a63dc3add4c828224a41a9cf9cf89 2024-12-05T05:17:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T05:17:44,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d86a9ecd60a645f2a7b9e8f55fae5546 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d86a9ecd60a645f2a7b9e8f55fae5546 2024-12-05T05:17:44,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e31e47e8efea4a6b9d218ea35c25924d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e31e47e8efea4a6b9d218ea35c25924d 2024-12-05T05:17:44,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97bc242f5c314ae7b43a1700cbe09631 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97bc242f5c314ae7b43a1700cbe09631 2024-12-05T05:17:44,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/9332e0e1302045fcb518885951706551 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/9332e0e1302045fcb518885951706551 2024-12-05T05:17:44,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/059aba8cab964060a64a1c7cc75f8cce to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/059aba8cab964060a64a1c7cc75f8cce 2024-12-05T05:17:44,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/1952548d9bb34aada1afe0bb629011fb to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/1952548d9bb34aada1afe0bb629011fb 2024-12-05T05:17:44,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e37f20149638410bb47e9782fff09b52 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e37f20149638410bb47e9782fff09b52 2024-12-05T05:17:44,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ebd9df7432a64ed68dab791d34ed26ae to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ebd9df7432a64ed68dab791d34ed26ae 2024-12-05T05:17:44,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c6d93bd62ec54cbe8b9fe149e921d847 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c6d93bd62ec54cbe8b9fe149e921d847 2024-12-05T05:17:44,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/58f4c2124cda42088ab8be8860da1184 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/58f4c2124cda42088ab8be8860da1184 2024-12-05T05:17:44,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c4ac8790c0d542dbbb99b2d78bea7309 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/c4ac8790c0d542dbbb99b2d78bea7309 2024-12-05T05:17:44,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/6a707db7706a42aa89f01564345fe355 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/6a707db7706a42aa89f01564345fe355 2024-12-05T05:17:44,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e54f7c54c86944e099e7f6f0f2ccbe12 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e54f7c54c86944e099e7f6f0f2ccbe12 2024-12-05T05:17:44,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/388da213f1874d2190afe3317598c09b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/388da213f1874d2190afe3317598c09b 2024-12-05T05:17:44,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/842a7ebd3a6d4696bfa6432ae4813b0a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/842a7ebd3a6d4696bfa6432ae4813b0a 2024-12-05T05:17:44,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e55f44cc77994a6e959d9cbc2743fc9f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/e55f44cc77994a6e959d9cbc2743fc9f 2024-12-05T05:17:44,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/10f5fea1d67f46c19d7cf3eecd0fb80f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/10f5fea1d67f46c19d7cf3eecd0fb80f 2024-12-05T05:17:44,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b9ff2d7866de4630b99b31a7c1c7b91e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b9ff2d7866de4630b99b31a7c1c7b91e 2024-12-05T05:17:44,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/38b22618d56d417cb213c73f2986c4f1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/38b22618d56d417cb213c73f2986c4f1 2024-12-05T05:17:44,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97ce021ceb934d0f909d92cc78c2c80d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/97ce021ceb934d0f909d92cc78c2c80d 2024-12-05T05:17:44,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d763521b952f4d57a28c088c667ba384 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/d763521b952f4d57a28c088c667ba384 2024-12-05T05:17:44,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/acffea6edb474a88ad7b1ea402023105 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/acffea6edb474a88ad7b1ea402023105 2024-12-05T05:17:44,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/dc1630a42def4f0186a95b5d9f399f23 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/dc1630a42def4f0186a95b5d9f399f23 2024-12-05T05:17:44,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ea838c4bfdf64724b3aa07c0d1073743 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/ea838c4bfdf64724b3aa07c0d1073743 2024-12-05T05:17:44,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/113e91d77a91414fa5ee7c1a79883259 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/113e91d77a91414fa5ee7c1a79883259 2024-12-05T05:17:44,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b59c501d12344c328a487881fa99fb21 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/b59c501d12344c328a487881fa99fb21 2024-12-05T05:17:44,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/62fbf3242f3143688d5d9775ee1189c6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/62fbf3242f3143688d5d9775ee1189c6 2024-12-05T05:17:44,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/eee04919226841b1b06edaef380d4d20 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/eee04919226841b1b06edaef380d4d20 2024-12-05T05:17:45,010 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/recovered.edits/464.seqid, newMaxSeqId=464, maxSeqId=1 2024-12-05T05:17:45,011 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. 2024-12-05T05:17:45,011 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for ac783a93e1fe6938afb704b21f8b8675: Waiting for close lock at 1733375863979Running coprocessor pre-close hooks at 1733375863979Disabling compacts and flushes for region at 1733375863979Disabling writes for close at 1733375863979Obtaining lock to block concurrent updates at 1733375863979Preparing flush snapshotting stores in ac783a93e1fe6938afb704b21f8b8675 at 1733375863979Finished memstore snapshotting TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733375863980 (+1 ms)Flushing stores of TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675. at 1733375863980Flushing ac783a93e1fe6938afb704b21f8b8675/A: creating writer at 1733375863980Flushing ac783a93e1fe6938afb704b21f8b8675/A: appending metadata at 1733375863983 (+3 ms)Flushing ac783a93e1fe6938afb704b21f8b8675/A: closing flushed file at 1733375863983Flushing ac783a93e1fe6938afb704b21f8b8675/B: creating writer at 1733375864394 (+411 ms)Flushing ac783a93e1fe6938afb704b21f8b8675/B: appending metadata at 1733375864397 (+3 ms)Flushing ac783a93e1fe6938afb704b21f8b8675/B: closing flushed file at 1733375864397Flushing ac783a93e1fe6938afb704b21f8b8675/C: creating writer at 1733375864413 (+16 ms)Flushing ac783a93e1fe6938afb704b21f8b8675/C: appending metadata at 1733375864417 (+4 ms)Flushing ac783a93e1fe6938afb704b21f8b8675/C: closing flushed file at 1733375864417Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70b0f74c: reopening flushed file at 1733375864829 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4869c029: reopening flushed file at 1733375864837 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7923e0a5: reopening flushed file at 1733375864842 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for ac783a93e1fe6938afb704b21f8b8675 in 868ms, sequenceid=461, compaction requested=false at 1733375864847 (+5 ms)Writing region close event to WAL at 1733375865001 (+154 ms)Running coprocessor post-close hooks at 1733375865011 (+10 ms)Closed at 1733375865011 2024-12-05T05:17:45,013 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:45,014 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=ac783a93e1fe6938afb704b21f8b8675, regionState=CLOSED 2024-12-05T05:17:45,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:17:45,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-05T05:17:45,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure ac783a93e1fe6938afb704b21f8b8675, server=8edcaf2ecc84,36233,1733375771401 in 1.1900 sec 2024-12-05T05:17:45,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-05T05:17:45,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac783a93e1fe6938afb704b21f8b8675, UNASSIGN in 1.1950 sec 2024-12-05T05:17:45,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-05T05:17:45,023 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1990 sec 2024-12-05T05:17:45,024 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375865024"}]},"ts":"1733375865024"} 2024-12-05T05:17:45,025 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T05:17:45,025 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T05:17:45,027 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2090 sec 2024-12-05T05:17:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-05T05:17:45,963 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:45,964 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T05:17:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:45,965 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=80, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T05:17:45,966 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=80, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:45,968 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:45,971 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/recovered.edits] 2024-12-05T05:17:45,974 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/58af783f680346d0846db2a0efa1df00 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/58af783f680346d0846db2a0efa1df00 2024-12-05T05:17:45,976 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ead6ed85bafe4d338c207f33103807c7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/A/ead6ed85bafe4d338c207f33103807c7 2024-12-05T05:17:45,978 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7f6d3cfff79644da8b0b698a989c30a3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/7f6d3cfff79644da8b0b698a989c30a3 2024-12-05T05:17:45,980 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/86719054505f431282784ca5e70ab078 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/B/86719054505f431282784ca5e70ab078 2024-12-05T05:17:45,982 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f922aca1cd3e45c2a37fa4c167e594a4 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/f922aca1cd3e45c2a37fa4c167e594a4 2024-12-05T05:17:45,983 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/fa31d32c630e4ee0ab57af1d11ba5da5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/C/fa31d32c630e4ee0ab57af1d11ba5da5 2024-12-05T05:17:45,985 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/recovered.edits/464.seqid to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675/recovered.edits/464.seqid 2024-12-05T05:17:45,985 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ac783a93e1fe6938afb704b21f8b8675 2024-12-05T05:17:45,985 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T05:17:45,987 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=80, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:45,989 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T05:17:45,991 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T05:17:45,992 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=80, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:45,992 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T05:17:45,992 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733375865992"}]},"ts":"9223372036854775807"} 2024-12-05T05:17:45,993 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T05:17:45,993 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ac783a93e1fe6938afb704b21f8b8675, NAME => 'TestAcidGuarantees,,1733375830603.ac783a93e1fe6938afb704b21f8b8675.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T05:17:45,993 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T05:17:45,993 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733375865993"}]},"ts":"9223372036854775807"} 2024-12-05T05:17:45,995 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T05:17:45,995 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=80, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:45,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-12-05T05:17:46,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-05T05:17:46,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T05:17:46,074 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:46,083 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 250), OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=616 (was 794), ProcessCount=11 (was 11), AvailableMemoryMB=3322 (was 3673) 2024-12-05T05:17:46,092 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=616, ProcessCount=11, AvailableMemoryMB=3320 2024-12-05T05:17:46,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:17:46,094 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:17:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:46,096 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T05:17:46,096 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:46,096 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 81 2024-12-05T05:17:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T05:17:46,097 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T05:17:46,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742165_1341 (size=1061) 2024-12-05T05:17:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T05:17:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T05:17:46,503 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 2024-12-05T05:17:46,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742166_1342 (size=53) 2024-12-05T05:17:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T05:17:46,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:17:46,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 04aadddcc47c85ef20a018672c5865b6, disabling compactions & flushes 2024-12-05T05:17:46,908 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:46,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:46,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. after waiting 0 ms 2024-12-05T05:17:46,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:46,908 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:46,908 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 04aadddcc47c85ef20a018672c5865b6: Waiting for close lock at 1733375866908Disabling compacts and flushes for region at 1733375866908Disabling writes for close at 1733375866908Writing region close event to WAL at 1733375866908Closed at 1733375866908 2024-12-05T05:17:46,909 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T05:17:46,909 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733375866909"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733375866909"}]},"ts":"1733375866909"} 2024-12-05T05:17:46,911 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T05:17:46,912 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T05:17:46,912 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375866912"}]},"ts":"1733375866912"} 2024-12-05T05:17:46,913 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T05:17:46,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, ASSIGN}] 2024-12-05T05:17:46,915 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, ASSIGN 2024-12-05T05:17:46,916 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, ASSIGN; state=OFFLINE, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=false 2024-12-05T05:17:47,066 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=04aadddcc47c85ef20a018672c5865b6, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:47,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, ASSIGN because future has completed 2024-12-05T05:17:47,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:17:47,223 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:47,223 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7752): Opening region: {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:17:47,223 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,223 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:17:47,223 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7794): checking encryption for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,223 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(7797): checking classloading for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,224 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,225 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:47,225 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04aadddcc47c85ef20a018672c5865b6 columnFamilyName A 2024-12-05T05:17:47,226 DEBUG [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:47,226 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(327): Store=04aadddcc47c85ef20a018672c5865b6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:47,226 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,227 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:47,227 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04aadddcc47c85ef20a018672c5865b6 columnFamilyName B 2024-12-05T05:17:47,227 DEBUG [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:47,228 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(327): Store=04aadddcc47c85ef20a018672c5865b6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:47,228 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,229 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:47,229 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04aadddcc47c85ef20a018672c5865b6 columnFamilyName C 2024-12-05T05:17:47,229 DEBUG [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:47,229 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(327): Store=04aadddcc47c85ef20a018672c5865b6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:47,229 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:47,229 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1038): replaying wal for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,230 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,230 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,231 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1048): stopping wal replay for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,231 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1060): Cleaning up temporary data for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,232 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:17:47,233 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1093): writing seq id for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T05:17:47,234 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T05:17:47,235 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1114): Opened 04aadddcc47c85ef20a018672c5865b6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67885565, jitterRate=0.011573746800422668}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:17:47,235 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:47,235 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegion(1006): Region open journal for 04aadddcc47c85ef20a018672c5865b6: Running coprocessor pre-open hook at 1733375867223Writing region info on filesystem at 1733375867223Initializing all the Stores at 1733375867224 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375867224Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375867224Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375867224Cleaning up temporary data from old regions at 1733375867231 (+7 ms)Running coprocessor post-open hooks at 1733375867235 (+4 ms)Region opened successfully at 1733375867235 2024-12-05T05:17:47,236 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., pid=83, masterSystemTime=1733375867220 2024-12-05T05:17:47,237 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:47,237 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=83}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:47,238 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=04aadddcc47c85ef20a018672c5865b6, regionState=OPEN, openSeqNum=2, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:47,239 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:17:47,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-05T05:17:47,241 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 in 172 msec 2024-12-05T05:17:47,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-05T05:17:47,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, ASSIGN in 327 msec 2024-12-05T05:17:47,244 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T05:17:47,244 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375867244"}]},"ts":"1733375867244"} 2024-12-05T05:17:47,245 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T05:17:47,246 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=81, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T05:17:47,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-12-05T05:17:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-05T05:17:48,244 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:48,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a96fba8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:48,245 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:48,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:48,247 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:48,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f8943f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:48,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:48,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:48,250 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:48,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59eda463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:48,252 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:48,256 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:48,257 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T05:17:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 8edcaf2ecc84,46695,1733375770505 2024-12-05T05:17:48,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ff0b568 2024-12-05T05:17:48,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T05:17:48,260 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T05:17:48,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:17:48,261 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:17:48,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T05:17:48,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742167_1343 (size=1097) 2024-12-05T05:17:48,672 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T05:17:48,672 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T05:17:48,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:17:48,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, REOPEN/MOVE}] 2024-12-05T05:17:48,678 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, REOPEN/MOVE 2024-12-05T05:17:48,679 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=04aadddcc47c85ef20a018672c5865b6, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:48,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, REOPEN/MOVE because future has completed 2024-12-05T05:17:48,680 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:17:48,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:17:48,833 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:48,833 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:17:48,834 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 04aadddcc47c85ef20a018672c5865b6, disabling compactions & flushes 2024-12-05T05:17:48,834 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:48,834 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:48,834 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. after waiting 0 ms 2024-12-05T05:17:48,834 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:48,841 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T05:17:48,842 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:48,842 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 04aadddcc47c85ef20a018672c5865b6: Waiting for close lock at 1733375868833Running coprocessor pre-close hooks at 1733375868833Disabling compacts and flushes for region at 1733375868833Disabling writes for close at 1733375868834 (+1 ms)Writing region close event to WAL at 1733375868835 (+1 ms)Running coprocessor post-close hooks at 1733375868842 (+7 ms)Closed at 1733375868842 2024-12-05T05:17:48,843 WARN [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionServer(3299): Not adding moved region record: 04aadddcc47c85ef20a018672c5865b6 to self. 2024-12-05T05:17:48,846 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:48,847 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=04aadddcc47c85ef20a018672c5865b6, regionState=CLOSED 2024-12-05T05:17:48,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:17:48,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-05T05:17:48,853 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 in 171 msec 2024-12-05T05:17:48,854 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, REOPEN/MOVE; state=CLOSED, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=true 2024-12-05T05:17:49,005 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=04aadddcc47c85ef20a018672c5865b6, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, REOPEN/MOVE because future has completed 2024-12-05T05:17:49,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:17:49,167 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,167 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7752): Opening region: {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:17:49,167 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,167 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:17:49,167 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7794): checking encryption for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,167 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(7797): checking classloading for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,169 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,170 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:49,170 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04aadddcc47c85ef20a018672c5865b6 columnFamilyName A 2024-12-05T05:17:49,171 DEBUG [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:49,171 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(327): Store=04aadddcc47c85ef20a018672c5865b6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:49,172 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,172 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:49,173 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04aadddcc47c85ef20a018672c5865b6 columnFamilyName B 2024-12-05T05:17:49,173 DEBUG [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:49,173 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(327): Store=04aadddcc47c85ef20a018672c5865b6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:49,173 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,173 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:17:49,174 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04aadddcc47c85ef20a018672c5865b6 columnFamilyName C 2024-12-05T05:17:49,174 DEBUG [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:49,174 INFO [StoreOpener-04aadddcc47c85ef20a018672c5865b6-1 {}] regionserver.HStore(327): Store=04aadddcc47c85ef20a018672c5865b6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:17:49,174 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,174 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1038): replaying wal for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,175 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,176 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,176 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1048): stopping wal replay for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,176 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1060): Cleaning up temporary data for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,177 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:17:49,178 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1093): writing seq id for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,179 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1114): Opened 04aadddcc47c85ef20a018672c5865b6; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71907420, jitterRate=0.07150405645370483}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:17:49,179 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,179 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegion(1006): Region open journal for 04aadddcc47c85ef20a018672c5865b6: Running coprocessor pre-open hook at 1733375869168Writing region info on filesystem at 1733375869168Initializing all the Stores at 1733375869168Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375869168Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375869169 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375869169Cleaning up temporary data from old regions at 1733375869176 (+7 ms)Running coprocessor post-open hooks at 1733375869179 (+3 ms)Region opened successfully at 1733375869179 2024-12-05T05:17:49,180 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., pid=88, masterSystemTime=1733375869163 2024-12-05T05:17:49,182 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,182 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=88}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,182 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=04aadddcc47c85ef20a018672c5865b6, regionState=OPEN, openSeqNum=5, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,183 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=86, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:17:49,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=86 2024-12-05T05:17:49,186 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=86, state=SUCCESS, hasLock=false; OpenRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 in 175 msec 2024-12-05T05:17:49,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-05T05:17:49,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, REOPEN/MOVE in 509 msec 2024-12-05T05:17:49,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-05T05:17:49,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 513 msec 2024-12-05T05:17:49,193 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 931 msec 2024-12-05T05:17:49,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-05T05:17:49,195 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:49,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a8a339, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,197 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b8a8dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,198 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,198 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f7b329c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f785f54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,200 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49852574, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,202 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5845c962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41b285eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,203 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,204 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180fcd42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,205 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@447f4c0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d7a2e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,207 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,207 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,207 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d82cc1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,209 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@92ce17e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,210 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d57dc3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,210 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,210 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,211 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fab29ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,212 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0e8d3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,214 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f33055, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,214 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,214 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,215 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f7e60db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,216 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ac3e55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,217 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bfb46db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,217 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,217 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,218 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c4bbd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,220 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63821016, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e87c032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f7c75e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@576d3c9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f344892, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,225 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@feb579e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,226 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,227 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52870, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,227 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b190b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e843cff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:17:49,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:17:49,229 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:17:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:17:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:17:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bbfe1b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:17:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:17:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:17:49,231 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:17:49,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16938d7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:17:49,232 DEBUG [Thread-1571 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,232 DEBUG [Thread-1578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,232 DEBUG [Thread-1567 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,232 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:49,233 DEBUG [Thread-1573 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,233 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,234 DEBUG [Thread-1569 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-12-05T05:17:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,235 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,235 DEBUG [Thread-1576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,236 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T05:17:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,239 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,243 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,244 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:49,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,246 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:17:49,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:49,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:49,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:49,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:49,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:49,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:49,252 DEBUG [Thread-1565 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,256 DEBUG [Thread-1580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:17:49,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:17:49,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:17:49,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:17:49,261 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:17:49,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205eda682839aee4c23ab05165f0b217e2e_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375869244/Put/seqid=0 2024-12-05T05:17:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375929280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375929280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375929282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375929282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375869265, max timestamp=1733375869279Num uniques 859; 2024-12-05T05:17:49,285 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:49,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375869265, max timestamp=1733375869279Num uniques 859; 2024-12-05T05:17:49,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375869265, max timestamp=1733375869279Num uniques 859; 2024-12-05T05:17:49,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:49,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:17:49,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742168_1344 (size=14594) 2024-12-05T05:17:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T05:17:49,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375929394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375929394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375929401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375929402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375929404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375929513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375929513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375929513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:49,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T05:17:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375929613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375929613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,698 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:49,702 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205eda682839aee4c23ab05165f0b217e2e_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eda682839aee4c23ab05165f0b217e2e_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:49,703 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e2cb8b41b7a44564aa66b09578cde7dc, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:49,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e2cb8b41b7a44564aa66b09578cde7dc is 175, key is test_row_0/A:col10/1733375869244/Put/seqid=0 2024-12-05T05:17:49,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:49,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742169_1345 (size=39549) 2024-12-05T05:17:49,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375929722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375929723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375929723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,732 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:49,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:49,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:49,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T05:17:49,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375929924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:49,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375929924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:49,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:50,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:50,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375930043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375930043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375930043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,123 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e2cb8b41b7a44564aa66b09578cde7dc 2024-12-05T05:17:50,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/1af21d16421e4605bd006e653765b620 is 50, key is test_row_0/B:col10/1733375869244/Put/seqid=0 2024-12-05T05:17:50,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742170_1346 (size=12001) 2024-12-05T05:17:50,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:50,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:50,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T05:17:50,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:50,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375930433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375930443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:50,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:50,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,537 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T05:17:50,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/1af21d16421e4605bd006e653765b620 2024-12-05T05:17:50,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:50,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375930553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375930553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:50,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375930553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:50,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/54d43dd89bcd4709a07ba1d1c17f802d is 50, key is test_row_0/C:col10/1733375869244/Put/seqid=0 2024-12-05T05:17:50,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742171_1347 (size=12001) 2024-12-05T05:17:50,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:50,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] handler.RSProcedureHandler(58): pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=90 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=90 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:50,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/54d43dd89bcd4709a07ba1d1c17f802d 2024-12-05T05:17:50,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e2cb8b41b7a44564aa66b09578cde7dc as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e2cb8b41b7a44564aa66b09578cde7dc 2024-12-05T05:17:50,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e2cb8b41b7a44564aa66b09578cde7dc, entries=200, sequenceid=14, filesize=38.6 K 2024-12-05T05:17:50,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/1af21d16421e4605bd006e653765b620 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/1af21d16421e4605bd006e653765b620 2024-12-05T05:17:51,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/1af21d16421e4605bd006e653765b620, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T05:17:51,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/54d43dd89bcd4709a07ba1d1c17f802d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/54d43dd89bcd4709a07ba1d1c17f802d 2024-12-05T05:17:51,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/54d43dd89bcd4709a07ba1d1c17f802d, entries=150, sequenceid=14, filesize=11.7 K 2024-12-05T05:17:51,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 04aadddcc47c85ef20a018672c5865b6 in 1758ms, sequenceid=14, compaction requested=false 2024-12-05T05:17:51,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:51,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-12-05T05:17:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:51,086 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-05T05:17:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:17:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120546fbae056c9d4e46bc723f68d7ff482e_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375869401/Put/seqid=0 2024-12-05T05:17:51,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742172_1348 (size=12154) 2024-12-05T05:17:51,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:51,146 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120546fbae056c9d4e46bc723f68d7ff482e_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120546fbae056c9d4e46bc723f68d7ff482e_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:51,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e7639264e8594eb9b154a20094b8af25, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:51,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e7639264e8594eb9b154a20094b8af25 is 175, key is test_row_0/A:col10/1733375869401/Put/seqid=0 2024-12-05T05:17:51,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742173_1349 (size=30955) 2024-12-05T05:17:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T05:17:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:51,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:51,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375931472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375931473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375931573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375931573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,578 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375931573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,578 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,579 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375931582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375931582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,589 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e7639264e8594eb9b154a20094b8af25 2024-12-05T05:17:51,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/d0dd75a50e3a4268b40bb81ba32f15e2 is 50, key is test_row_0/B:col10/1733375869401/Put/seqid=0 2024-12-05T05:17:51,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742174_1350 (size=12001) 2024-12-05T05:17:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375931793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375931794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:51,999 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/d0dd75a50e3a4268b40bb81ba32f15e2 2024-12-05T05:17:52,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/26a39eb5cd3c41f88196611ba686a01b is 50, key is test_row_0/C:col10/1733375869401/Put/seqid=0 2024-12-05T05:17:52,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742175_1351 (size=12001) 2024-12-05T05:17:52,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375932113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:52,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375932113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:52,410 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/26a39eb5cd3c41f88196611ba686a01b 2024-12-05T05:17:52,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e7639264e8594eb9b154a20094b8af25 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e7639264e8594eb9b154a20094b8af25 2024-12-05T05:17:52,433 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e7639264e8594eb9b154a20094b8af25, entries=150, sequenceid=51, filesize=30.2 K 2024-12-05T05:17:52,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/d0dd75a50e3a4268b40bb81ba32f15e2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d0dd75a50e3a4268b40bb81ba32f15e2 2024-12-05T05:17:52,451 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d0dd75a50e3a4268b40bb81ba32f15e2, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T05:17:52,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/26a39eb5cd3c41f88196611ba686a01b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/26a39eb5cd3c41f88196611ba686a01b 2024-12-05T05:17:52,467 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/26a39eb5cd3c41f88196611ba686a01b, entries=150, sequenceid=51, filesize=11.7 K 2024-12-05T05:17:52,468 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 04aadddcc47c85ef20a018672c5865b6 in 1382ms, sequenceid=51, compaction requested=false 2024-12-05T05:17:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:52,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-05T05:17:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-05T05:17:52,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-05T05:17:52,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2290 sec 2024-12-05T05:17:52,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 3.2430 sec 2024-12-05T05:17:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:52,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:17:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:52,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:52,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050d3ae972bb5b4a78aec7084834f1e260_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375872624/Put/seqid=0 2024-12-05T05:17:52,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742176_1352 (size=17034) 2024-12-05T05:17:52,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:52,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375932710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:52,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:52,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375932720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:52,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:52,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375932823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,829 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375932832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:52,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375933032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375933043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,049 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:53,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050d3ae972bb5b4a78aec7084834f1e260_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050d3ae972bb5b4a78aec7084834f1e260_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:53,065 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/6c51ca5cf9bc4170925600cdaceba652, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:53,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/6c51ca5cf9bc4170925600cdaceba652 is 175, key is test_row_0/A:col10/1733375872624/Put/seqid=0 2024-12-05T05:17:53,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742177_1353 (size=48139) 2024-12-05T05:17:53,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375933343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375933352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-05T05:17:53,393 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:53,393 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:53,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-12-05T05:17:53,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T05:17:53,395 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:53,396 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:53,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:53,469 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/6c51ca5cf9bc4170925600cdaceba652 2024-12-05T05:17:53,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/109cfe1a56164be0ac17f6878e5d961f is 50, key is test_row_0/B:col10/1733375872624/Put/seqid=0 2024-12-05T05:17:53,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742178_1354 (size=12001) 2024-12-05T05:17:53,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/109cfe1a56164be0ac17f6878e5d961f 2024-12-05T05:17:53,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/7bfc6dec89dc4226a4633e9b77de2a85 is 50, key is test_row_0/C:col10/1733375872624/Put/seqid=0 2024-12-05T05:17:53,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T05:17:53,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742179_1355 (size=12001) 2024-12-05T05:17:53,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T05:17:53,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:53,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375933582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,592 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375933593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375933593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,601 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,602 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T05:17:53,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:53,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T05:17:53,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T05:17:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375933853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:53,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:53,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375933864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:53,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:53,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/7bfc6dec89dc4226a4633e9b77de2a85 2024-12-05T05:17:53,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/6c51ca5cf9bc4170925600cdaceba652 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/6c51ca5cf9bc4170925600cdaceba652 2024-12-05T05:17:53,919 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/6c51ca5cf9bc4170925600cdaceba652, entries=250, sequenceid=63, filesize=47.0 K 2024-12-05T05:17:53,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/109cfe1a56164be0ac17f6878e5d961f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/109cfe1a56164be0ac17f6878e5d961f 2024-12-05T05:17:53,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/109cfe1a56164be0ac17f6878e5d961f, entries=150, sequenceid=63, filesize=11.7 K 2024-12-05T05:17:53,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/7bfc6dec89dc4226a4633e9b77de2a85 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/7bfc6dec89dc4226a4633e9b77de2a85 2024-12-05T05:17:53,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/7bfc6dec89dc4226a4633e9b77de2a85, entries=150, sequenceid=63, filesize=11.7 K 2024-12-05T05:17:53,928 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 04aadddcc47c85ef20a018672c5865b6 in 1304ms, sequenceid=63, compaction requested=true 2024-12-05T05:17:53,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:53,928 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:53,929 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:53,930 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:53,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:53,931 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:53,931 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/A is initiating minor compaction (all files) 2024-12-05T05:17:53,931 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/A in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,931 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:53,931 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/B is initiating minor compaction (all files) 2024-12-05T05:17:53,931 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e2cb8b41b7a44564aa66b09578cde7dc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e7639264e8594eb9b154a20094b8af25, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/6c51ca5cf9bc4170925600cdaceba652] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=115.9 K 2024-12-05T05:17:53,931 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/B in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,931 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:53,931 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/1af21d16421e4605bd006e653765b620, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d0dd75a50e3a4268b40bb81ba32f15e2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/109cfe1a56164be0ac17f6878e5d961f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=35.2 K 2024-12-05T05:17:53,931 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e2cb8b41b7a44564aa66b09578cde7dc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e7639264e8594eb9b154a20094b8af25, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/6c51ca5cf9bc4170925600cdaceba652] 2024-12-05T05:17:53,931 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 1af21d16421e4605bd006e653765b620, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375869238 2024-12-05T05:17:53,931 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2cb8b41b7a44564aa66b09578cde7dc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375869238 2024-12-05T05:17:53,932 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d0dd75a50e3a4268b40bb81ba32f15e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733375869395 2024-12-05T05:17:53,932 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7639264e8594eb9b154a20094b8af25, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733375869395 2024-12-05T05:17:53,932 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 109cfe1a56164be0ac17f6878e5d961f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375871467 2024-12-05T05:17:53,932 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c51ca5cf9bc4170925600cdaceba652, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375871466 2024-12-05T05:17:53,941 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:53,941 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#B#compaction#303 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:53,942 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/f05efcc37df54f5a8573ddb342e99dd3 is 50, key is test_row_0/B:col10/1733375872624/Put/seqid=0 2024-12-05T05:17:53,945 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120550a2dd1071ee4b149655fd7895b2400c_04aadddcc47c85ef20a018672c5865b6 store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:53,947 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120550a2dd1071ee4b149655fd7895b2400c_04aadddcc47c85ef20a018672c5865b6, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:53,947 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120550a2dd1071ee4b149655fd7895b2400c_04aadddcc47c85ef20a018672c5865b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:53,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742180_1356 (size=12104) 2024-12-05T05:17:53,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742181_1357 (size=4469) 2024-12-05T05:17:54,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-12-05T05:17:54,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:54,013 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T05:17:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:54,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:54,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120548c6d0f64c234127b003abefdb01703a_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375872708/Put/seqid=0 2024-12-05T05:17:54,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T05:17:54,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742182_1358 (size=12154) 2024-12-05T05:17:54,362 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:54,363 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/f05efcc37df54f5a8573ddb342e99dd3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f05efcc37df54f5a8573ddb342e99dd3 2024-12-05T05:17:54,363 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/a55bcb7b9ac540edb7efc38a6100239b is 175, key is test_row_0/A:col10/1733375872624/Put/seqid=0 2024-12-05T05:17:54,371 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/B of 04aadddcc47c85ef20a018672c5865b6 into f05efcc37df54f5a8573ddb342e99dd3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:54,371 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:54,371 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/B, priority=13, startTime=1733375873930; duration=0sec 2024-12-05T05:17:54,371 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:54,371 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:B 2024-12-05T05:17:54,372 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:54,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742183_1359 (size=31058) 2024-12-05T05:17:54,373 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:54,373 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/C is initiating minor compaction (all files) 2024-12-05T05:17:54,373 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/C in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:54,373 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/54d43dd89bcd4709a07ba1d1c17f802d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/26a39eb5cd3c41f88196611ba686a01b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/7bfc6dec89dc4226a4633e9b77de2a85] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=35.2 K 2024-12-05T05:17:54,374 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 54d43dd89bcd4709a07ba1d1c17f802d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1733375869238 2024-12-05T05:17:54,374 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 26a39eb5cd3c41f88196611ba686a01b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1733375869395 2024-12-05T05:17:54,374 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bfc6dec89dc4226a4633e9b77de2a85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375871467 2024-12-05T05:17:54,383 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/a55bcb7b9ac540edb7efc38a6100239b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/a55bcb7b9ac540edb7efc38a6100239b 2024-12-05T05:17:54,388 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#C#compaction#306 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:54,389 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/283cd3a453144252bf4fe0885233b6d2 is 50, key is test_row_0/C:col10/1733375872624/Put/seqid=0 2024-12-05T05:17:54,395 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/A of 04aadddcc47c85ef20a018672c5865b6 into a55bcb7b9ac540edb7efc38a6100239b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:54,395 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:54,395 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/A, priority=13, startTime=1733375873928; duration=0sec 2024-12-05T05:17:54,395 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:54,395 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:A 2024-12-05T05:17:54,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742184_1360 (size=12104) 2024-12-05T05:17:54,423 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/283cd3a453144252bf4fe0885233b6d2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/283cd3a453144252bf4fe0885233b6d2 2024-12-05T05:17:54,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:54,433 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/C of 04aadddcc47c85ef20a018672c5865b6 into 283cd3a453144252bf4fe0885233b6d2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:54,433 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:54,433 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/C, priority=13, startTime=1733375873930; duration=0sec 2024-12-05T05:17:54,433 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:54,433 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:C 2024-12-05T05:17:54,438 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120548c6d0f64c234127b003abefdb01703a_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120548c6d0f64c234127b003abefdb01703a_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:54,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/4d88c25edc254ca5a7226b0d59b3b49d, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:54,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/4d88c25edc254ca5a7226b0d59b3b49d is 175, key is test_row_0/A:col10/1733375872708/Put/seqid=0 2024-12-05T05:17:54,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742185_1361 (size=30955) 2024-12-05T05:17:54,458 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/4d88c25edc254ca5a7226b0d59b3b49d 2024-12-05T05:17:54,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/583b719f0db744ad8e8c45d930d49d87 is 50, key is test_row_0/B:col10/1733375872708/Put/seqid=0 2024-12-05T05:17:54,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742186_1362 (size=12001) 2024-12-05T05:17:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T05:17:54,870 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/583b719f0db744ad8e8c45d930d49d87 2024-12-05T05:17:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:54,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:54,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/56671c08e6194475839232c7a8298d32 is 50, key is test_row_0/C:col10/1733375872708/Put/seqid=0 2024-12-05T05:17:54,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742187_1363 (size=12001) 2024-12-05T05:17:54,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375934913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:54,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375934916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:54,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375935022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375935033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375935236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375935243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,295 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/56671c08e6194475839232c7a8298d32 2024-12-05T05:17:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/4d88c25edc254ca5a7226b0d59b3b49d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4d88c25edc254ca5a7226b0d59b3b49d 2024-12-05T05:17:55,309 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4d88c25edc254ca5a7226b0d59b3b49d, entries=150, sequenceid=83, filesize=30.2 K 2024-12-05T05:17:55,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/583b719f0db744ad8e8c45d930d49d87 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/583b719f0db744ad8e8c45d930d49d87 2024-12-05T05:17:55,317 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/583b719f0db744ad8e8c45d930d49d87, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T05:17:55,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/56671c08e6194475839232c7a8298d32 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/56671c08e6194475839232c7a8298d32 2024-12-05T05:17:55,327 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/56671c08e6194475839232c7a8298d32, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T05:17:55,329 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 04aadddcc47c85ef20a018672c5865b6 in 1317ms, sequenceid=83, compaction requested=false 2024-12-05T05:17:55,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:55,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:55,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-05T05:17:55,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-12-05T05:17:55,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-05T05:17:55,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9360 sec 2024-12-05T05:17:55,339 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.9450 sec 2024-12-05T05:17:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-05T05:17:55,543 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:55,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:55,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-12-05T05:17:55,546 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:55,547 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:55,547 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:55,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:17:55,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:55,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:55,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:55,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:55,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:55,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T05:17:55,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120598d9551035a9487fbdaef34e51aad1c1_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375875546/Put/seqid=0 2024-12-05T05:17:55,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742188_1364 (size=12154) 2024-12-05T05:17:55,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:55,599 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120598d9551035a9487fbdaef34e51aad1c1_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120598d9551035a9487fbdaef34e51aad1c1_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:55,600 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/90997718a0a946b387b45d179169b1b7, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:55,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/90997718a0a946b387b45d179169b1b7 is 175, key is test_row_0/A:col10/1733375875546/Put/seqid=0 2024-12-05T05:17:55,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742189_1365 (size=30955) 2024-12-05T05:17:55,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375935625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375935629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T05:17:55,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T05:17:55,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:55,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:55,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:55,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:55,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:55,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375935743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375935744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,747 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T05:17:55,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:55,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:55,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:55,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:55,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:55,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:55,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T05:17:55,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375935953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375935953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:55,959 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:55,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:56,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T05:17:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,029 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=99, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/90997718a0a946b387b45d179169b1b7 2024-12-05T05:17:56,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/ffe9686a3b6943e898b3200460e91424 is 50, key is test_row_0/B:col10/1733375875546/Put/seqid=0 2024-12-05T05:17:56,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742190_1366 (size=12001) 2024-12-05T05:17:56,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/ffe9686a3b6943e898b3200460e91424 2024-12-05T05:17:56,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/5d4282d70451428fb41abc51fa4dab7f is 50, key is test_row_0/C:col10/1733375875546/Put/seqid=0 2024-12-05T05:17:56,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T05:17:56,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:56,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742191_1367 (size=12001) 2024-12-05T05:17:56,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T05:17:56,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375936263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:56,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:56,267 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:56,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375936273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:56,282 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:56,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T05:17:56,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:56,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T05:17:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:56,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=99 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/5d4282d70451428fb41abc51fa4dab7f 2024-12-05T05:17:56,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/90997718a0a946b387b45d179169b1b7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90997718a0a946b387b45d179169b1b7 2024-12-05T05:17:56,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90997718a0a946b387b45d179169b1b7, entries=150, sequenceid=99, filesize=30.2 K 2024-12-05T05:17:56,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/ffe9686a3b6943e898b3200460e91424 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ffe9686a3b6943e898b3200460e91424 2024-12-05T05:17:56,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ffe9686a3b6943e898b3200460e91424, entries=150, sequenceid=99, filesize=11.7 K 2024-12-05T05:17:56,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/5d4282d70451428fb41abc51fa4dab7f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5d4282d70451428fb41abc51fa4dab7f 2024-12-05T05:17:56,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5d4282d70451428fb41abc51fa4dab7f, entries=150, sequenceid=99, filesize=11.7 K 2024-12-05T05:17:56,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 04aadddcc47c85ef20a018672c5865b6 in 1038ms, sequenceid=99, compaction requested=true 2024-12-05T05:17:56,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:56,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:56,586 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:56,586 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:56,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:56,587 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:56,587 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/A is initiating minor compaction (all files) 2024-12-05T05:17:56,587 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/A in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,587 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/a55bcb7b9ac540edb7efc38a6100239b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4d88c25edc254ca5a7226b0d59b3b49d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90997718a0a946b387b45d179169b1b7] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=90.8 K 2024-12-05T05:17:56,587 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,587 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/a55bcb7b9ac540edb7efc38a6100239b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4d88c25edc254ca5a7226b0d59b3b49d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90997718a0a946b387b45d179169b1b7] 2024-12-05T05:17:56,591 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a55bcb7b9ac540edb7efc38a6100239b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375871467 2024-12-05T05:17:56,591 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d88c25edc254ca5a7226b0d59b3b49d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375872663 2024-12-05T05:17:56,591 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:56,591 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/B is initiating minor compaction (all files) 2024-12-05T05:17:56,592 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/B in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,592 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90997718a0a946b387b45d179169b1b7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733375874888 2024-12-05T05:17:56,592 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f05efcc37df54f5a8573ddb342e99dd3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/583b719f0db744ad8e8c45d930d49d87, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ffe9686a3b6943e898b3200460e91424] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=35.3 K 2024-12-05T05:17:56,592 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f05efcc37df54f5a8573ddb342e99dd3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375871467 2024-12-05T05:17:56,593 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 583b719f0db744ad8e8c45d930d49d87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375872663 2024-12-05T05:17:56,593 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe9686a3b6943e898b3200460e91424, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733375874888 2024-12-05T05:17:56,601 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:56,605 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:56,605 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/0014a34c84c9449aac0ff9eff4f5cc54 is 50, key is test_row_0/B:col10/1733375875546/Put/seqid=0 2024-12-05T05:17:56,617 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412050cca88fbb1444f7f9957c8e9b90fed53_04aadddcc47c85ef20a018672c5865b6 store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:56,618 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412050cca88fbb1444f7f9957c8e9b90fed53_04aadddcc47c85ef20a018672c5865b6, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:56,619 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050cca88fbb1444f7f9957c8e9b90fed53_04aadddcc47c85ef20a018672c5865b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:56,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-12-05T05:17:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,622 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:17:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:56,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:56,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742192_1368 (size=12207) 2024-12-05T05:17:56,670 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/0014a34c84c9449aac0ff9eff4f5cc54 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0014a34c84c9449aac0ff9eff4f5cc54 2024-12-05T05:17:56,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059d9019effde94ff9ab392f162704e048_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375875606/Put/seqid=0 2024-12-05T05:17:56,682 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/B of 04aadddcc47c85ef20a018672c5865b6 into 0014a34c84c9449aac0ff9eff4f5cc54(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:56,682 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:56,682 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/B, priority=13, startTime=1733375876586; duration=0sec 2024-12-05T05:17:56,682 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:56,682 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:B 2024-12-05T05:17:56,683 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T05:17:56,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742193_1369 (size=4469) 2024-12-05T05:17:56,685 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:56,686 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/C is initiating minor compaction (all files) 2024-12-05T05:17:56,686 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#A#compaction#312 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:56,686 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/C in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,686 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/283cd3a453144252bf4fe0885233b6d2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/56671c08e6194475839232c7a8298d32, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5d4282d70451428fb41abc51fa4dab7f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=35.3 K 2024-12-05T05:17:56,686 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/12450a7f6b404bb1802259e39d761e2a is 175, key is test_row_0/A:col10/1733375875546/Put/seqid=0 2024-12-05T05:17:56,686 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 283cd3a453144252bf4fe0885233b6d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733375871467 2024-12-05T05:17:56,687 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 56671c08e6194475839232c7a8298d32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375872663 2024-12-05T05:17:56,688 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d4282d70451428fb41abc51fa4dab7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733375874888 2024-12-05T05:17:56,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742194_1370 (size=12154) 2024-12-05T05:17:56,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:56,712 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059d9019effde94ff9ab392f162704e048_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059d9019effde94ff9ab392f162704e048_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:56,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/69dc0f076c1641a4b81e52f4e6777da7, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:56,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/69dc0f076c1641a4b81e52f4e6777da7 is 175, key is test_row_0/A:col10/1733375875606/Put/seqid=0 2024-12-05T05:17:56,744 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:56,745 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/6274a27ea5eb48c8868178a64a3f00e3 is 50, key is test_row_0/C:col10/1733375875546/Put/seqid=0 2024-12-05T05:17:56,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742195_1371 (size=31161) 2024-12-05T05:17:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:56,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:56,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742196_1372 (size=30955) 2024-12-05T05:17:56,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742197_1373 (size=12207) 2024-12-05T05:17:56,798 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/6274a27ea5eb48c8868178a64a3f00e3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6274a27ea5eb48c8868178a64a3f00e3 2024-12-05T05:17:56,798 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/69dc0f076c1641a4b81e52f4e6777da7 2024-12-05T05:17:56,813 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/C of 04aadddcc47c85ef20a018672c5865b6 into 6274a27ea5eb48c8868178a64a3f00e3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:56,813 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:56,813 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/C, priority=13, startTime=1733375876586; duration=0sec 2024-12-05T05:17:56,813 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:56,813 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:C 2024-12-05T05:17:56,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/304fb357bf59415984bf66c54a24f7e6 is 50, key is test_row_0/B:col10/1733375875606/Put/seqid=0 2024-12-05T05:17:56,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742198_1374 (size=12001) 2024-12-05T05:17:56,854 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/304fb357bf59415984bf66c54a24f7e6 2024-12-05T05:17:56,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c8bfe4437ef647f49aa40fa5c1d97f4a is 50, key is test_row_0/C:col10/1733375875606/Put/seqid=0 2024-12-05T05:17:56,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375936865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:56,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:56,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375936868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:56,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:56,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:56,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742199_1375 (size=12001) 2024-12-05T05:17:56,905 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c8bfe4437ef647f49aa40fa5c1d97f4a 2024-12-05T05:17:56,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/69dc0f076c1641a4b81e52f4e6777da7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/69dc0f076c1641a4b81e52f4e6777da7 2024-12-05T05:17:56,923 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/69dc0f076c1641a4b81e52f4e6777da7, entries=150, sequenceid=118, filesize=30.2 K 2024-12-05T05:17:56,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/304fb357bf59415984bf66c54a24f7e6 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/304fb357bf59415984bf66c54a24f7e6 2024-12-05T05:17:56,931 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/304fb357bf59415984bf66c54a24f7e6, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T05:17:56,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c8bfe4437ef647f49aa40fa5c1d97f4a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8bfe4437ef647f49aa40fa5c1d97f4a 2024-12-05T05:17:56,937 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8bfe4437ef647f49aa40fa5c1d97f4a, entries=150, sequenceid=118, filesize=11.7 K 2024-12-05T05:17:56,938 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 04aadddcc47c85ef20a018672c5865b6 in 316ms, sequenceid=118, compaction requested=false 2024-12-05T05:17:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:56,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-05T05:17:56,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-05T05:17:56,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-05T05:17:56,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3930 sec 2024-12-05T05:17:56,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.4010 sec 2024-12-05T05:17:56,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:56,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T05:17:56,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:56,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:56,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:56,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:56,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:56,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:57,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053adc010760694e36b069922a67d91bcc_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375876785/Put/seqid=0 2024-12-05T05:17:57,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742200_1376 (size=14744) 2024-12-05T05:17:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375937082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375937089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,154 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/12450a7f6b404bb1802259e39d761e2a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/12450a7f6b404bb1802259e39d761e2a 2024-12-05T05:17:57,158 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/A of 04aadddcc47c85ef20a018672c5865b6 into 12450a7f6b404bb1802259e39d761e2a(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:57,158 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:57,158 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/A, priority=13, startTime=1733375876585; duration=0sec 2024-12-05T05:17:57,158 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:57,158 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:A 2024-12-05T05:17:57,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375937203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375937203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375937413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375937414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,443 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:57,457 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053adc010760694e36b069922a67d91bcc_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053adc010760694e36b069922a67d91bcc_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:57,458 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7b54b5dd43e244ceaa9496a94c80ddad, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:57,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7b54b5dd43e244ceaa9496a94c80ddad is 175, key is test_row_0/A:col10/1733375876785/Put/seqid=0 2024-12-05T05:17:57,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742201_1377 (size=39699) 2024-12-05T05:17:57,496 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7b54b5dd43e244ceaa9496a94c80ddad 2024-12-05T05:17:57,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3 is 50, key is test_row_0/B:col10/1733375876785/Put/seqid=0 2024-12-05T05:17:57,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742202_1378 (size=12101) 2024-12-05T05:17:57,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375937613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,625 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:57,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375937625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,626 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375937633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,639 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:17:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-05T05:17:57,705 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:57,706 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-12-05T05:17:57,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T05:17:57,708 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:57,708 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:57,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:57,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375937723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:57,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375937725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:57,730 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T05:17:57,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T05:17:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:57,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:57,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:57,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3 2024-12-05T05:17:57,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/46e25febf03d4687b94d95cdd2826a58 is 50, key is test_row_0/C:col10/1733375876785/Put/seqid=0 2024-12-05T05:17:57,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742203_1379 (size=12101) 2024-12-05T05:17:57,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/46e25febf03d4687b94d95cdd2826a58 2024-12-05T05:17:58,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7b54b5dd43e244ceaa9496a94c80ddad as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7b54b5dd43e244ceaa9496a94c80ddad 2024-12-05T05:17:58,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7b54b5dd43e244ceaa9496a94c80ddad, entries=200, sequenceid=134, filesize=38.8 K 2024-12-05T05:17:58,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3 2024-12-05T05:17:58,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T05:17:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:58,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:17:58,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T05:17:58,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3, entries=150, sequenceid=134, filesize=11.8 K 2024-12-05T05:17:58,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/46e25febf03d4687b94d95cdd2826a58 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/46e25febf03d4687b94d95cdd2826a58 2024-12-05T05:17:58,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/46e25febf03d4687b94d95cdd2826a58, entries=150, sequenceid=134, filesize=11.8 K 2024-12-05T05:17:58,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 04aadddcc47c85ef20a018672c5865b6 in 1051ms, sequenceid=134, compaction requested=true 2024-12-05T05:17:58,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:58,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:17:58,041 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:58,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:58,041 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:58,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:17:58,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:58,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:17:58,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:58,042 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:58,042 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/A is initiating minor compaction (all files) 2024-12-05T05:17:58,042 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/A in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:58,042 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/12450a7f6b404bb1802259e39d761e2a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/69dc0f076c1641a4b81e52f4e6777da7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7b54b5dd43e244ceaa9496a94c80ddad] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=99.4 K 2024-12-05T05:17:58,042 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:58,042 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/12450a7f6b404bb1802259e39d761e2a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/69dc0f076c1641a4b81e52f4e6777da7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7b54b5dd43e244ceaa9496a94c80ddad] 2024-12-05T05:17:58,042 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:58,042 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/B is initiating minor compaction (all files) 2024-12-05T05:17:58,042 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/B in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:58,042 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0014a34c84c9449aac0ff9eff4f5cc54, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/304fb357bf59415984bf66c54a24f7e6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=35.5 K 2024-12-05T05:17:58,043 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12450a7f6b404bb1802259e39d761e2a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733375874888 2024-12-05T05:17:58,043 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0014a34c84c9449aac0ff9eff4f5cc54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733375874888 2024-12-05T05:17:58,043 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69dc0f076c1641a4b81e52f4e6777da7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733375875606 2024-12-05T05:17:58,043 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 304fb357bf59415984bf66c54a24f7e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733375875606 2024-12-05T05:17:58,043 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b54b5dd43e244ceaa9496a94c80ddad, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733375876785 2024-12-05T05:17:58,044 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting dc1d4e4424ba4cb3b2e84a07aefa6cd3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733375876785 2024-12-05T05:17:58,070 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#B#compaction#321 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:58,070 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/21733c6a1fc946e295aa57ce702448bc is 50, key is test_row_0/B:col10/1733375876785/Put/seqid=0 2024-12-05T05:17:58,077 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:58,079 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205d073569cf05e4731b86f2856077d34dc_04aadddcc47c85ef20a018672c5865b6 store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:58,082 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205d073569cf05e4731b86f2856077d34dc_04aadddcc47c85ef20a018672c5865b6, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:58,082 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205d073569cf05e4731b86f2856077d34dc_04aadddcc47c85ef20a018672c5865b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:58,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742204_1380 (size=12409) 2024-12-05T05:17:58,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742205_1381 (size=4469) 2024-12-05T05:17:58,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-12-05T05:17:58,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:58,170 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T05:17:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:58,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:58,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055149b92cc1cb473dae8c6976bcd7722c_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375877088/Put/seqid=0 2024-12-05T05:17:58,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742206_1382 (size=12304) 2024-12-05T05:17:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:58,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:17:58,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375938292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:58,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:58,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375938292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:58,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T05:17:58,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:58,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375938507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:58,515 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/21733c6a1fc946e295aa57ce702448bc as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/21733c6a1fc946e295aa57ce702448bc 2024-12-05T05:17:58,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:58,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375938507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:58,519 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/B of 04aadddcc47c85ef20a018672c5865b6 into 21733c6a1fc946e295aa57ce702448bc(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:58,519 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:58,519 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/B, priority=13, startTime=1733375878041; duration=0sec 2024-12-05T05:17:58,519 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:17:58,519 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:B 2024-12-05T05:17:58,519 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:17:58,520 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:17:58,520 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/C is initiating minor compaction (all files) 2024-12-05T05:17:58,520 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/C in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:58,520 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6274a27ea5eb48c8868178a64a3f00e3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8bfe4437ef647f49aa40fa5c1d97f4a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/46e25febf03d4687b94d95cdd2826a58] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=35.5 K 2024-12-05T05:17:58,520 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6274a27ea5eb48c8868178a64a3f00e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=99, earliestPutTs=1733375874888 2024-12-05T05:17:58,520 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c8bfe4437ef647f49aa40fa5c1d97f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1733375875606 2024-12-05T05:17:58,521 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e25febf03d4687b94d95cdd2826a58, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733375876785 2024-12-05T05:17:58,525 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#C#compaction#324 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:58,526 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/b944f8ecf3f34cc1a0b3a7837be333a9 is 50, key is test_row_0/C:col10/1733375876785/Put/seqid=0 2024-12-05T05:17:58,531 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#A#compaction#322 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:17:58,531 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/913f3b5dacbe48acaf20b3bef9f31151 is 175, key is test_row_0/A:col10/1733375876785/Put/seqid=0 2024-12-05T05:17:58,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742207_1383 (size=12409) 2024-12-05T05:17:58,541 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/b944f8ecf3f34cc1a0b3a7837be333a9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/b944f8ecf3f34cc1a0b3a7837be333a9 2024-12-05T05:17:58,545 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/C of 04aadddcc47c85ef20a018672c5865b6 into b944f8ecf3f34cc1a0b3a7837be333a9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:58,545 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:58,545 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/C, priority=13, startTime=1733375878041; duration=0sec 2024-12-05T05:17:58,545 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:58,545 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:C 2024-12-05T05:17:58,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742208_1384 (size=31363) 2024-12-05T05:17:58,549 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/913f3b5dacbe48acaf20b3bef9f31151 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/913f3b5dacbe48acaf20b3bef9f31151 2024-12-05T05:17:58,553 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/A of 04aadddcc47c85ef20a018672c5865b6 into 913f3b5dacbe48acaf20b3bef9f31151(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:17:58,553 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:58,553 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/A, priority=13, startTime=1733375878041; duration=0sec 2024-12-05T05:17:58,553 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:17:58,553 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:A 2024-12-05T05:17:58,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:58,628 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055149b92cc1cb473dae8c6976bcd7722c_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055149b92cc1cb473dae8c6976bcd7722c_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:58,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/48b8aa967f5940e0b1a30a457a660680, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:58,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/48b8aa967f5940e0b1a30a457a660680 is 175, key is test_row_0/A:col10/1733375877088/Put/seqid=0 2024-12-05T05:17:58,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742209_1385 (size=31105) 2024-12-05T05:17:58,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:58,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375938712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375938723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T05:17:59,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:59,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375939022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:59,036 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/48b8aa967f5940e0b1a30a457a660680 2024-12-05T05:17:59,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375939032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:59,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/0cbd6af9e7d64f0595abb14e59f4c9e1 is 50, key is test_row_0/B:col10/1733375877088/Put/seqid=0 2024-12-05T05:17:59,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742210_1386 (size=12151) 2024-12-05T05:17:59,049 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/0cbd6af9e7d64f0595abb14e59f4c9e1 2024-12-05T05:17:59,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c08e10b79e64459d858914ed0dabfe6f is 50, key is test_row_0/C:col10/1733375877088/Put/seqid=0 2024-12-05T05:17:59,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742211_1387 (size=12151) 2024-12-05T05:17:59,097 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c08e10b79e64459d858914ed0dabfe6f 2024-12-05T05:17:59,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/48b8aa967f5940e0b1a30a457a660680 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/48b8aa967f5940e0b1a30a457a660680 2024-12-05T05:17:59,105 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/48b8aa967f5940e0b1a30a457a660680, entries=150, sequenceid=153, filesize=30.4 K 2024-12-05T05:17:59,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/0cbd6af9e7d64f0595abb14e59f4c9e1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0cbd6af9e7d64f0595abb14e59f4c9e1 2024-12-05T05:17:59,115 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0cbd6af9e7d64f0595abb14e59f4c9e1, entries=150, sequenceid=153, filesize=11.9 K 2024-12-05T05:17:59,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c08e10b79e64459d858914ed0dabfe6f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c08e10b79e64459d858914ed0dabfe6f 2024-12-05T05:17:59,123 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c08e10b79e64459d858914ed0dabfe6f, entries=150, sequenceid=153, filesize=11.9 K 2024-12-05T05:17:59,126 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 04aadddcc47c85ef20a018672c5865b6 in 956ms, sequenceid=153, compaction requested=false 2024-12-05T05:17:59,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:17:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:17:59,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-05T05:17:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-05T05:17:59,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-05T05:17:59,138 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-12-05T05:17:59,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.4330 sec 2024-12-05T05:17:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:59,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T05:17:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:17:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:17:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:17:59,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:17:59,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120593b82a08275c4330bfc4d2567b2150da_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375879534/Put/seqid=0 2024-12-05T05:17:59,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742212_1388 (size=14794) 2024-12-05T05:17:59,559 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:17:59,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120593b82a08275c4330bfc4d2567b2150da_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120593b82a08275c4330bfc4d2567b2150da_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:17:59,568 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/73312bf237e14f38ae5d1c278b2b228d, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:17:59,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/73312bf237e14f38ae5d1c278b2b228d is 175, key is test_row_0/A:col10/1733375879534/Put/seqid=0 2024-12-05T05:17:59,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742213_1389 (size=39749) 2024-12-05T05:17:59,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:59,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375939615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,637 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:59,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375939629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:59,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:59,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375939743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:59,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:59,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:59,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375939753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-05T05:17:59,853 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:17:59,854 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:17:59,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-12-05T05:17:59,856 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:17:59,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T05:17:59,856 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:17:59,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:17:59,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375939955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:17:59,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T05:17:59,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:17:59,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375939974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:17:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:17:59,981 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:00,009 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/73312bf237e14f38ae5d1c278b2b228d 2024-12-05T05:18:00,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T05:18:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:00,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/a2c803b08eae498a89420fc894342245 is 50, key is test_row_0/B:col10/1733375879534/Put/seqid=0 2024-12-05T05:18:00,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742214_1390 (size=12151) 2024-12-05T05:18:00,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T05:18:00,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:00,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T05:18:00,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:00,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375940273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:00,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:00,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375940292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:00,302 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:00,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:00,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T05:18:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/a2c803b08eae498a89420fc894342245 2024-12-05T05:18:00,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T05:18:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:00,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/157b292513414743a469f2cab9fe4ee8 is 50, key is test_row_0/C:col10/1733375879534/Put/seqid=0 2024-12-05T05:18:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T05:18:00,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742215_1391 (size=12151) 2024-12-05T05:18:00,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/157b292513414743a469f2cab9fe4ee8 2024-12-05T05:18:00,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/73312bf237e14f38ae5d1c278b2b228d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/73312bf237e14f38ae5d1c278b2b228d 2024-12-05T05:18:00,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/73312bf237e14f38ae5d1c278b2b228d, entries=200, sequenceid=171, filesize=38.8 K 2024-12-05T05:18:00,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/a2c803b08eae498a89420fc894342245 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/a2c803b08eae498a89420fc894342245 2024-12-05T05:18:00,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/a2c803b08eae498a89420fc894342245, entries=150, sequenceid=171, filesize=11.9 K 2024-12-05T05:18:00,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/157b292513414743a469f2cab9fe4ee8 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/157b292513414743a469f2cab9fe4ee8 2024-12-05T05:18:00,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/157b292513414743a469f2cab9fe4ee8, entries=150, sequenceid=171, filesize=11.9 K 2024-12-05T05:18:00,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 04aadddcc47c85ef20a018672c5865b6 in 979ms, sequenceid=171, compaction requested=true 2024-12-05T05:18:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:00,515 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:00,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:00,515 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102217 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/B is initiating minor compaction (all files) 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/A is initiating minor compaction (all files) 2024-12-05T05:18:00,516 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/B in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,516 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/A in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,516 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/21733c6a1fc946e295aa57ce702448bc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0cbd6af9e7d64f0595abb14e59f4c9e1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/a2c803b08eae498a89420fc894342245] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=35.9 K 2024-12-05T05:18:00,516 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/913f3b5dacbe48acaf20b3bef9f31151, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/48b8aa967f5940e0b1a30a457a660680, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/73312bf237e14f38ae5d1c278b2b228d] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=99.8 K 2024-12-05T05:18:00,516 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/913f3b5dacbe48acaf20b3bef9f31151, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/48b8aa967f5940e0b1a30a457a660680, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/73312bf237e14f38ae5d1c278b2b228d] 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 21733c6a1fc946e295aa57ce702448bc, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733375876785 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 913f3b5dacbe48acaf20b3bef9f31151, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733375876785 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cbd6af9e7d64f0595abb14e59f4c9e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733375877056 2024-12-05T05:18:00,516 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48b8aa967f5940e0b1a30a457a660680, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733375877056 2024-12-05T05:18:00,517 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a2c803b08eae498a89420fc894342245, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375878280 2024-12-05T05:18:00,517 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73312bf237e14f38ae5d1c278b2b228d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375878278 2024-12-05T05:18:00,522 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:00,522 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/cc9f3275a151465d91b282868c209a21 is 50, key is test_row_0/B:col10/1733375879534/Put/seqid=0 2024-12-05T05:18:00,523 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:00,524 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205c35fd2a1a0dd4fabbf2e303f9f0e8612_04aadddcc47c85ef20a018672c5865b6 store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:00,526 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205c35fd2a1a0dd4fabbf2e303f9f0e8612_04aadddcc47c85ef20a018672c5865b6, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:00,526 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c35fd2a1a0dd4fabbf2e303f9f0e8612_04aadddcc47c85ef20a018672c5865b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:00,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742216_1392 (size=12561) 2024-12-05T05:18:00,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742217_1393 (size=4469) 2024-12-05T05:18:00,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-12-05T05:18:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,626 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T05:18:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:00,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:00,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053d13526732de4f01be37ce7cfbc606bc_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375879627/Put/seqid=0 2024-12-05T05:18:00,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742218_1394 (size=12304) 2024-12-05T05:18:00,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:00,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:00,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375940891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375940890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:00,934 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#A#compaction#331 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:00,934 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/90e5cb133eaf48638b822d8b75626bad is 175, key is test_row_0/A:col10/1733375879534/Put/seqid=0 2024-12-05T05:18:00,937 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/cc9f3275a151465d91b282868c209a21 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/cc9f3275a151465d91b282868c209a21 2024-12-05T05:18:00,943 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/B of 04aadddcc47c85ef20a018672c5865b6 into cc9f3275a151465d91b282868c209a21(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:00,944 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:00,944 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/B, priority=13, startTime=1733375880515; duration=0sec 2024-12-05T05:18:00,944 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:00,944 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:B 2024-12-05T05:18:00,944 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:00,945 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:00,946 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/C is initiating minor compaction (all files) 2024-12-05T05:18:00,946 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/C in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:00,946 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/b944f8ecf3f34cc1a0b3a7837be333a9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c08e10b79e64459d858914ed0dabfe6f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/157b292513414743a469f2cab9fe4ee8] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=35.9 K 2024-12-05T05:18:00,946 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b944f8ecf3f34cc1a0b3a7837be333a9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1733375876785 2024-12-05T05:18:00,947 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c08e10b79e64459d858914ed0dabfe6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1733375877056 2024-12-05T05:18:00,947 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 157b292513414743a469f2cab9fe4ee8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375878280 2024-12-05T05:18:00,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742219_1395 (size=31515) 2024-12-05T05:18:00,978 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:00,979 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/10eb269e5a9e4707a02cf83d874b8868 is 50, key is test_row_0/C:col10/1733375879534/Put/seqid=0 2024-12-05T05:18:00,981 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/90e5cb133eaf48638b822d8b75626bad as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90e5cb133eaf48638b822d8b75626bad 2024-12-05T05:18:00,991 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/A of 04aadddcc47c85ef20a018672c5865b6 into 90e5cb133eaf48638b822d8b75626bad(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:00,992 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:00,992 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/A, priority=13, startTime=1733375880515; duration=0sec 2024-12-05T05:18:00,992 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:00,992 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:A 2024-12-05T05:18:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T05:18:01,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742220_1396 (size=12561) 2024-12-05T05:18:01,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375941013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375941013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:01,051 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053d13526732de4f01be37ce7cfbc606bc_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053d13526732de4f01be37ce7cfbc606bc_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:01,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/f4d85087461e4f058c36d094b8dfc200, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:01,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/f4d85087461e4f058c36d094b8dfc200 is 175, key is test_row_0/A:col10/1733375879627/Put/seqid=0 2024-12-05T05:18:01,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742221_1397 (size=31105) 2024-12-05T05:18:01,070 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/f4d85087461e4f058c36d094b8dfc200 2024-12-05T05:18:01,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/62483fc4d9e9467a91e0d9c1cd1f4f08 is 50, key is test_row_0/B:col10/1733375879627/Put/seqid=0 2024-12-05T05:18:01,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742222_1398 (size=12151) 2024-12-05T05:18:01,122 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/62483fc4d9e9467a91e0d9c1cd1f4f08 2024-12-05T05:18:01,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/3a0586ec7a4e472294f9c92b68c950ab is 50, key is test_row_0/C:col10/1733375879627/Put/seqid=0 2024-12-05T05:18:01,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742223_1399 (size=12151) 2024-12-05T05:18:01,146 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/3a0586ec7a4e472294f9c92b68c950ab 2024-12-05T05:18:01,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/f4d85087461e4f058c36d094b8dfc200 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/f4d85087461e4f058c36d094b8dfc200 2024-12-05T05:18:01,163 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/f4d85087461e4f058c36d094b8dfc200, entries=150, sequenceid=188, filesize=30.4 K 2024-12-05T05:18:01,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/62483fc4d9e9467a91e0d9c1cd1f4f08 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/62483fc4d9e9467a91e0d9c1cd1f4f08 2024-12-05T05:18:01,170 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/62483fc4d9e9467a91e0d9c1cd1f4f08, entries=150, sequenceid=188, filesize=11.9 K 2024-12-05T05:18:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/3a0586ec7a4e472294f9c92b68c950ab as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3a0586ec7a4e472294f9c92b68c950ab 2024-12-05T05:18:01,175 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3a0586ec7a4e472294f9c92b68c950ab, entries=150, sequenceid=188, filesize=11.9 K 2024-12-05T05:18:01,177 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 04aadddcc47c85ef20a018672c5865b6 in 552ms, sequenceid=188, compaction requested=false 2024-12-05T05:18:01,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:01,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:01,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-05T05:18:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-05T05:18:01,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-05T05:18:01,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3230 sec 2024-12-05T05:18:01,183 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.3270 sec 2024-12-05T05:18:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:01,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T05:18:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:01,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:01,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120598ffaba893184c419641a6fbe5c520bb_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375880886/Put/seqid=0 2024-12-05T05:18:01,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742224_1400 (size=17284) 2024-12-05T05:18:01,242 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:01,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120598ffaba893184c419641a6fbe5c520bb_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120598ffaba893184c419641a6fbe5c520bb_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:01,248 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/ed66bbe1cc5943369c71c787628fb71c, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:01,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/ed66bbe1cc5943369c71c787628fb71c is 175, key is test_row_0/A:col10/1733375880886/Put/seqid=0 2024-12-05T05:18:01,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742225_1401 (size=48389) 2024-12-05T05:18:01,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375941294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375941295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,303 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375941413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375941413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,421 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/10eb269e5a9e4707a02cf83d874b8868 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/10eb269e5a9e4707a02cf83d874b8868 2024-12-05T05:18:01,425 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/C of 04aadddcc47c85ef20a018672c5865b6 into 10eb269e5a9e4707a02cf83d874b8868(size=12.3 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:01,425 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:01,425 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/C, priority=13, startTime=1733375880515; duration=0sec 2024-12-05T05:18:01,425 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:01,425 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:C 2024-12-05T05:18:01,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375941622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375941622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,627 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,654 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/ed66bbe1cc5943369c71c787628fb71c 2024-12-05T05:18:01,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/c7da41bf11b849d296cae815ee352f0e is 50, key is test_row_0/B:col10/1733375880886/Put/seqid=0 2024-12-05T05:18:01,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742226_1402 (size=12151) 2024-12-05T05:18:01,667 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/c7da41bf11b849d296cae815ee352f0e 2024-12-05T05:18:01,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/ab8adefc39f749089da13b5b2cb48572 is 50, key is test_row_0/C:col10/1733375880886/Put/seqid=0 2024-12-05T05:18:01,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742227_1403 (size=12151) 2024-12-05T05:18:01,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375941933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375941933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-05T05:18:02,003 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:02,003 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-05T05:18:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T05:18:02,006 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:02,006 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:02,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:02,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/ab8adefc39f749089da13b5b2cb48572 2024-12-05T05:18:02,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/ed66bbe1cc5943369c71c787628fb71c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/ed66bbe1cc5943369c71c787628fb71c 2024-12-05T05:18:02,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/ed66bbe1cc5943369c71c787628fb71c, entries=250, sequenceid=208, filesize=47.3 K 2024-12-05T05:18:02,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/c7da41bf11b849d296cae815ee352f0e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/c7da41bf11b849d296cae815ee352f0e 2024-12-05T05:18:02,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/c7da41bf11b849d296cae815ee352f0e, entries=150, sequenceid=208, filesize=11.9 K 2024-12-05T05:18:02,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/ab8adefc39f749089da13b5b2cb48572 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/ab8adefc39f749089da13b5b2cb48572 2024-12-05T05:18:02,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/ab8adefc39f749089da13b5b2cb48572, entries=150, sequenceid=208, filesize=11.9 K 2024-12-05T05:18:02,106 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 04aadddcc47c85ef20a018672c5865b6 in 879ms, sequenceid=208, compaction requested=true 2024-12-05T05:18:02,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:02,107 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:02,107 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:02,107 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:02,107 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:02,108 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/B is initiating minor compaction (all files) 2024-12-05T05:18:02,108 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/B in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:02,108 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/cc9f3275a151465d91b282868c209a21, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/62483fc4d9e9467a91e0d9c1cd1f4f08, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/c7da41bf11b849d296cae815ee352f0e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=36.0 K 2024-12-05T05:18:02,108 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:02,108 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/A is initiating minor compaction (all files) 2024-12-05T05:18:02,108 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/A in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:02,108 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90e5cb133eaf48638b822d8b75626bad, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/f4d85087461e4f058c36d094b8dfc200, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/ed66bbe1cc5943369c71c787628fb71c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=108.4 K 2024-12-05T05:18:02,108 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:02,108 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90e5cb133eaf48638b822d8b75626bad, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/f4d85087461e4f058c36d094b8dfc200, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/ed66bbe1cc5943369c71c787628fb71c] 2024-12-05T05:18:02,110 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9f3275a151465d91b282868c209a21, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375878280 2024-12-05T05:18:02,111 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90e5cb133eaf48638b822d8b75626bad, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375878280 2024-12-05T05:18:02,111 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4d85087461e4f058c36d094b8dfc200, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733375879583 2024-12-05T05:18:02,111 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 62483fc4d9e9467a91e0d9c1cd1f4f08, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733375879583 2024-12-05T05:18:02,111 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c7da41bf11b849d296cae815ee352f0e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733375880838 2024-12-05T05:18:02,111 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed66bbe1cc5943369c71c787628fb71c, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733375880838 2024-12-05T05:18:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T05:18:02,124 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:02,125 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:02,125 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/2ba4420114694c5e8da6f46123c791a3 is 50, key is test_row_0/B:col10/1733375880886/Put/seqid=0 2024-12-05T05:18:02,140 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412053a6d328e41ad4be394aff9a4584723f9_04aadddcc47c85ef20a018672c5865b6 store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:02,142 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412053a6d328e41ad4be394aff9a4584723f9_04aadddcc47c85ef20a018672c5865b6, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:02,142 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053a6d328e41ad4be394aff9a4584723f9_04aadddcc47c85ef20a018672c5865b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:02,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742228_1404 (size=12663) 2024-12-05T05:18:02,156 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/2ba4420114694c5e8da6f46123c791a3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2ba4420114694c5e8da6f46123c791a3 2024-12-05T05:18:02,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-05T05:18:02,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:02,158 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T05:18:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:02,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742229_1405 (size=4469) 2024-12-05T05:18:02,166 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/B of 04aadddcc47c85ef20a018672c5865b6 into 2ba4420114694c5e8da6f46123c791a3(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:02,166 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:02,166 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/B, priority=13, startTime=1733375882107; duration=0sec 2024-12-05T05:18:02,166 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:02,166 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:B 2024-12-05T05:18:02,166 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:02,167 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:02,167 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/C is initiating minor compaction (all files) 2024-12-05T05:18:02,167 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/C in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:02,167 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/10eb269e5a9e4707a02cf83d874b8868, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3a0586ec7a4e472294f9c92b68c950ab, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/ab8adefc39f749089da13b5b2cb48572] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=36.0 K 2024-12-05T05:18:02,168 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 10eb269e5a9e4707a02cf83d874b8868, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733375878280 2024-12-05T05:18:02,168 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a0586ec7a4e472294f9c92b68c950ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1733375879583 2024-12-05T05:18:02,168 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ab8adefc39f749089da13b5b2cb48572, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733375880838 2024-12-05T05:18:02,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b8738ca440df41ca89111ce3e0f42787_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375881294/Put/seqid=0 2024-12-05T05:18:02,186 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#C#compaction#342 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:02,186 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/3f8bf4ba40e14625acb2b3192e7218f5 is 50, key is test_row_0/C:col10/1733375880886/Put/seqid=0 2024-12-05T05:18:02,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742230_1406 (size=12304) 2024-12-05T05:18:02,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:02,190 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205b8738ca440df41ca89111ce3e0f42787_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b8738ca440df41ca89111ce3e0f42787_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:02,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742231_1407 (size=12663) 2024-12-05T05:18:02,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/08f1a5f65f094dbd9973b800962a1452, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:02,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/08f1a5f65f094dbd9973b800962a1452 is 175, key is test_row_0/A:col10/1733375881294/Put/seqid=0 2024-12-05T05:18:02,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742232_1408 (size=31105) 2024-12-05T05:18:02,196 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/08f1a5f65f094dbd9973b800962a1452 2024-12-05T05:18:02,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/b064056162fd4a6a93a3c7c215eff80f is 50, key is test_row_0/B:col10/1733375881294/Put/seqid=0 2024-12-05T05:18:02,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742233_1409 (size=12151) 2024-12-05T05:18:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T05:18:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:02,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:02,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375942492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,499 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:02,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:02,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375942497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:02,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:02,561 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:02,562 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7f3134ad1aa345be8891aa5a43a51259 is 175, key is test_row_0/A:col10/1733375880886/Put/seqid=0 2024-12-05T05:18:02,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742234_1410 (size=31617) 2024-12-05T05:18:02,574 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7f3134ad1aa345be8891aa5a43a51259 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7f3134ad1aa345be8891aa5a43a51259 2024-12-05T05:18:02,588 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/A of 04aadddcc47c85ef20a018672c5865b6 into 7f3134ad1aa345be8891aa5a43a51259(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:02,588 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:02,588 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/A, priority=13, startTime=1733375882106; duration=0sec 2024-12-05T05:18:02,588 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:02,588 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:A 2024-12-05T05:18:02,600 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/3f8bf4ba40e14625acb2b3192e7218f5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3f8bf4ba40e14625acb2b3192e7218f5 2024-12-05T05:18:02,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375942603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:02,607 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/C of 04aadddcc47c85ef20a018672c5865b6 into 3f8bf4ba40e14625acb2b3192e7218f5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:02,607 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:02,607 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/C, priority=13, startTime=1733375882107; duration=0sec 2024-12-05T05:18:02,607 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:02,607 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:C 2024-12-05T05:18:02,614 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/b064056162fd4a6a93a3c7c215eff80f 2024-12-05T05:18:02,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/a359bbc597354b68a9019ae97c56fc43 is 50, key is test_row_0/C:col10/1733375881294/Put/seqid=0 2024-12-05T05:18:02,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375942613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,624 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:02,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T05:18:02,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742235_1411 (size=12151) 2024-12-05T05:18:02,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:02,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375942812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:02,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:02,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375942833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,041 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/a359bbc597354b68a9019ae97c56fc43 2024-12-05T05:18:03,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/08f1a5f65f094dbd9973b800962a1452 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/08f1a5f65f094dbd9973b800962a1452 2024-12-05T05:18:03,050 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/08f1a5f65f094dbd9973b800962a1452, entries=150, sequenceid=223, filesize=30.4 K 2024-12-05T05:18:03,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/b064056162fd4a6a93a3c7c215eff80f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/b064056162fd4a6a93a3c7c215eff80f 2024-12-05T05:18:03,063 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/b064056162fd4a6a93a3c7c215eff80f, entries=150, sequenceid=223, filesize=11.9 K 2024-12-05T05:18:03,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/a359bbc597354b68a9019ae97c56fc43 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/a359bbc597354b68a9019ae97c56fc43 2024-12-05T05:18:03,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/a359bbc597354b68a9019ae97c56fc43, entries=150, sequenceid=223, filesize=11.9 K 2024-12-05T05:18:03,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 04aadddcc47c85ef20a018672c5865b6 in 929ms, sequenceid=223, compaction requested=false 2024-12-05T05:18:03,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:03,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-05T05:18:03,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-05T05:18:03,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-05T05:18:03,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0830 sec 2024-12-05T05:18:03,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.0880 sec 2024-12-05T05:18:03,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:03,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:18:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:03,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c517b48414804e41a289859eb8b02012_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375883135/Put/seqid=0 2024-12-05T05:18:03,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-05T05:18:03,155 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:03,155 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-05T05:18:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T05:18:03,157 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:03,157 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:03,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:03,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742236_1412 (size=14794) 2024-12-05T05:18:03,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:03,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375943201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375943204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T05:18:03,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:03,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375943323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:03,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:03,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375943324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T05:18:03,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:03,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:03,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375943544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375943548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,558 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,576 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:03,592 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c517b48414804e41a289859eb8b02012_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c517b48414804e41a289859eb8b02012_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:03,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/3718e4edfe514600b3c76aa94285a6e9, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:03,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/3718e4edfe514600b3c76aa94285a6e9 is 175, key is test_row_0/A:col10/1733375883135/Put/seqid=0 2024-12-05T05:18:03,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742237_1413 (size=39749) 2024-12-05T05:18:03,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T05:18:03,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:03,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375943863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:03,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375943863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:03,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:03,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:03,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:03,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:03,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,053 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/3718e4edfe514600b3c76aa94285a6e9 2024-12-05T05:18:04,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/3fe8fba0d93a49d18349909cdc857436 is 50, key is test_row_0/B:col10/1733375883135/Put/seqid=0 2024-12-05T05:18:04,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742238_1414 (size=12151) 2024-12-05T05:18:04,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T05:18:04,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:04,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:04,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375944383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:04,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:04,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:04,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375944383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:04,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:04,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:04,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:04,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/3fe8fba0d93a49d18349909cdc857436 2024-12-05T05:18:04,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/0adb547e50d7437088881ecaf4b46401 is 50, key is test_row_0/C:col10/1733375883135/Put/seqid=0 2024-12-05T05:18:04,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742239_1415 (size=12151) 2024-12-05T05:18:04,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:04,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:04,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:04,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:04,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:04,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/0adb547e50d7437088881ecaf4b46401 2024-12-05T05:18:04,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/3718e4edfe514600b3c76aa94285a6e9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/3718e4edfe514600b3c76aa94285a6e9 2024-12-05T05:18:04,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/3718e4edfe514600b3c76aa94285a6e9, entries=200, sequenceid=245, filesize=38.8 K 2024-12-05T05:18:04,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/3fe8fba0d93a49d18349909cdc857436 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/3fe8fba0d93a49d18349909cdc857436 2024-12-05T05:18:04,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/3fe8fba0d93a49d18349909cdc857436, entries=150, sequenceid=245, filesize=11.9 K 2024-12-05T05:18:04,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/0adb547e50d7437088881ecaf4b46401 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/0adb547e50d7437088881ecaf4b46401 2024-12-05T05:18:04,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/0adb547e50d7437088881ecaf4b46401, entries=150, sequenceid=245, filesize=11.9 K 2024-12-05T05:18:04,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 04aadddcc47c85ef20a018672c5865b6 in 1841ms, sequenceid=245, compaction requested=true 2024-12-05T05:18:04,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:04,978 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:04,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:04,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:04,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:04,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:04,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:04,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-05T05:18:04,978 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:04,979 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:04,979 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/A is initiating minor compaction (all files) 2024-12-05T05:18:04,979 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/A in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,979 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7f3134ad1aa345be8891aa5a43a51259, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/08f1a5f65f094dbd9973b800962a1452, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/3718e4edfe514600b3c76aa94285a6e9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=100.1 K 2024-12-05T05:18:04,979 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,979 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7f3134ad1aa345be8891aa5a43a51259, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/08f1a5f65f094dbd9973b800962a1452, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/3718e4edfe514600b3c76aa94285a6e9] 2024-12-05T05:18:04,980 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:04,980 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/B is initiating minor compaction (all files) 2024-12-05T05:18:04,980 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f3134ad1aa345be8891aa5a43a51259, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733375880838 2024-12-05T05:18:04,980 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/B in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:04,980 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2ba4420114694c5e8da6f46123c791a3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/b064056162fd4a6a93a3c7c215eff80f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/3fe8fba0d93a49d18349909cdc857436] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=36.1 K 2024-12-05T05:18:04,981 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08f1a5f65f094dbd9973b800962a1452, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733375881284 2024-12-05T05:18:04,981 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ba4420114694c5e8da6f46123c791a3, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733375880838 2024-12-05T05:18:04,981 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3718e4edfe514600b3c76aa94285a6e9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733375882485 2024-12-05T05:18:04,981 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b064056162fd4a6a93a3c7c215eff80f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733375881284 2024-12-05T05:18:04,982 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fe8fba0d93a49d18349909cdc857436, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733375882491 2024-12-05T05:18:04,989 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:04,992 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#B#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:04,993 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/ca9aecfcaaf04c98b8e1cf627d48c1d8 is 50, key is test_row_0/B:col10/1733375883135/Put/seqid=0 2024-12-05T05:18:05,004 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205595f7cc735774ab48813d1c0f6efbb90_04aadddcc47c85ef20a018672c5865b6 store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:05,005 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205595f7cc735774ab48813d1c0f6efbb90_04aadddcc47c85ef20a018672c5865b6, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:05,006 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205595f7cc735774ab48813d1c0f6efbb90_04aadddcc47c85ef20a018672c5865b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742240_1416 (size=12765) 2024-12-05T05:18:05,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742241_1417 (size=4469) 2024-12-05T05:18:05,069 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/ca9aecfcaaf04c98b8e1cf627d48c1d8 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ca9aecfcaaf04c98b8e1cf627d48c1d8 2024-12-05T05:18:05,077 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/B of 04aadddcc47c85ef20a018672c5865b6 into ca9aecfcaaf04c98b8e1cf627d48c1d8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:05,077 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:05,077 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/B, priority=13, startTime=1733375884978; duration=0sec 2024-12-05T05:18:05,077 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:05,077 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:B 2024-12-05T05:18:05,078 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:05,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-05T05:18:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:05,080 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:18:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:05,080 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:05,080 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/C is initiating minor compaction (all files) 2024-12-05T05:18:05,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:05,080 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/C in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:05,081 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3f8bf4ba40e14625acb2b3192e7218f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/a359bbc597354b68a9019ae97c56fc43, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/0adb547e50d7437088881ecaf4b46401] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=36.1 K 2024-12-05T05:18:05,081 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f8bf4ba40e14625acb2b3192e7218f5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1733375880838 2024-12-05T05:18:05,082 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a359bbc597354b68a9019ae97c56fc43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733375881284 2024-12-05T05:18:05,082 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0adb547e50d7437088881ecaf4b46401, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733375882491 2024-12-05T05:18:05,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059b9ab83682694d7b85242d277ee7365e_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375883191/Put/seqid=0 2024-12-05T05:18:05,094 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#C#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:05,094 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/6b0b51beb8bd4a5db673ef0f5d16706f is 50, key is test_row_0/C:col10/1733375883135/Put/seqid=0 2024-12-05T05:18:05,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742243_1419 (size=12765) 2024-12-05T05:18:05,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742242_1418 (size=12354) 2024-12-05T05:18:05,154 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/6b0b51beb8bd4a5db673ef0f5d16706f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6b0b51beb8bd4a5db673ef0f5d16706f 2024-12-05T05:18:05,162 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/C of 04aadddcc47c85ef20a018672c5865b6 into 6b0b51beb8bd4a5db673ef0f5d16706f(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:05,162 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:05,162 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/C, priority=13, startTime=1733375884978; duration=0sec 2024-12-05T05:18:05,162 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:05,162 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:C 2024-12-05T05:18:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T05:18:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:05,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:05,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375945451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:05,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375945452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:05,463 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#A#compaction#348 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:05,464 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/aeb64cadd3864f5290f9ac0e37f08e48 is 175, key is test_row_0/A:col10/1733375883135/Put/seqid=0 2024-12-05T05:18:05,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742244_1420 (size=31719) 2024-12-05T05:18:05,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:05,551 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059b9ab83682694d7b85242d277ee7365e_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b9ab83682694d7b85242d277ee7365e_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/fda179d558c642ab883743f97b9a17c7, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:05,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/fda179d558c642ab883743f97b9a17c7 is 175, key is test_row_0/A:col10/1733375883191/Put/seqid=0 2024-12-05T05:18:05,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742245_1421 (size=31155) 2024-12-05T05:18:05,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375945562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:05,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375945562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:05,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:05,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375945773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375945774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:05,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:05,879 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/aeb64cadd3864f5290f9ac0e37f08e48 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/aeb64cadd3864f5290f9ac0e37f08e48 2024-12-05T05:18:05,891 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/A of 04aadddcc47c85ef20a018672c5865b6 into aeb64cadd3864f5290f9ac0e37f08e48(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:05,891 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:05,891 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/A, priority=13, startTime=1733375884978; duration=0sec 2024-12-05T05:18:05,892 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:05,892 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:A 2024-12-05T05:18:05,955 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/fda179d558c642ab883743f97b9a17c7 2024-12-05T05:18:05,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/6ac919a03f3b4fc5a451860b8e53104a is 50, key is test_row_0/B:col10/1733375883191/Put/seqid=0 2024-12-05T05:18:05,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742246_1422 (size=12201) 2024-12-05T05:18:05,993 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/6ac919a03f3b4fc5a451860b8e53104a 2024-12-05T05:18:06,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/99c4a00dea3544689cec46c0051ed32b is 50, key is test_row_0/C:col10/1733375883191/Put/seqid=0 2024-12-05T05:18:06,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742247_1423 (size=12201) 2024-12-05T05:18:06,034 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/99c4a00dea3544689cec46c0051ed32b 2024-12-05T05:18:06,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/fda179d558c642ab883743f97b9a17c7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/fda179d558c642ab883743f97b9a17c7 2024-12-05T05:18:06,048 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/fda179d558c642ab883743f97b9a17c7, entries=150, sequenceid=259, filesize=30.4 K 2024-12-05T05:18:06,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/6ac919a03f3b4fc5a451860b8e53104a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/6ac919a03f3b4fc5a451860b8e53104a 2024-12-05T05:18:06,053 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/6ac919a03f3b4fc5a451860b8e53104a, entries=150, sequenceid=259, filesize=11.9 K 2024-12-05T05:18:06,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/99c4a00dea3544689cec46c0051ed32b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/99c4a00dea3544689cec46c0051ed32b 2024-12-05T05:18:06,059 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/99c4a00dea3544689cec46c0051ed32b, entries=150, sequenceid=259, filesize=11.9 K 2024-12-05T05:18:06,066 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 04aadddcc47c85ef20a018672c5865b6 in 986ms, sequenceid=259, compaction requested=false 2024-12-05T05:18:06,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:06,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:06,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-05T05:18:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-05T05:18:06,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-05T05:18:06,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9110 sec 2024-12-05T05:18:06,072 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.9160 sec 2024-12-05T05:18:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:06,087 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-05T05:18:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:06,087 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:06,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055ae2cf60e07c4c9f8e8dee1b9a699e13_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375885451/Put/seqid=0 2024-12-05T05:18:06,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742248_1424 (size=14994) 2024-12-05T05:18:06,127 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:06,130 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055ae2cf60e07c4c9f8e8dee1b9a699e13_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ae2cf60e07c4c9f8e8dee1b9a699e13_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:06,131 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/43b89764b7db456f80350d90521a772b, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:06,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/43b89764b7db456f80350d90521a772b is 175, key is test_row_0/A:col10/1733375885451/Put/seqid=0 2024-12-05T05:18:06,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375946144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:06,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:06,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375946150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,153 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:06,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742249_1425 (size=39949) 2024-12-05T05:18:06,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375946264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:06,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375946265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,273 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:06,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375946483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:06,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:06,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375946483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,493 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:06,559 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/43b89764b7db456f80350d90521a772b 2024-12-05T05:18:06,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/f8772cabbeb446e0a993bb02ddad73fe is 50, key is test_row_0/B:col10/1733375885451/Put/seqid=0 2024-12-05T05:18:06,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742250_1426 (size=12301) 2024-12-05T05:18:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375946802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:06,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375946802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:06,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/f8772cabbeb446e0a993bb02ddad73fe 2024-12-05T05:18:06,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/73c7ccef230f44259f93a135c2629b5a is 50, key is test_row_0/C:col10/1733375885451/Put/seqid=0 2024-12-05T05:18:06,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742251_1427 (size=12301) 2024-12-05T05:18:06,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/73c7ccef230f44259f93a135c2629b5a 2024-12-05T05:18:07,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/43b89764b7db456f80350d90521a772b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/43b89764b7db456f80350d90521a772b 2024-12-05T05:18:07,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/43b89764b7db456f80350d90521a772b, entries=200, sequenceid=281, filesize=39.0 K 2024-12-05T05:18:07,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/f8772cabbeb446e0a993bb02ddad73fe as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f8772cabbeb446e0a993bb02ddad73fe 2024-12-05T05:18:07,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f8772cabbeb446e0a993bb02ddad73fe, entries=150, sequenceid=281, filesize=12.0 K 2024-12-05T05:18:07,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/73c7ccef230f44259f93a135c2629b5a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/73c7ccef230f44259f93a135c2629b5a 2024-12-05T05:18:07,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/73c7ccef230f44259f93a135c2629b5a, entries=150, sequenceid=281, filesize=12.0 K 2024-12-05T05:18:07,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 04aadddcc47c85ef20a018672c5865b6 in 939ms, sequenceid=281, compaction requested=true 2024-12-05T05:18:07,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:07,026 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:07,026 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:07,028 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:07,028 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:07,028 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/B is initiating minor compaction (all files) 2024-12-05T05:18:07,028 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/A is initiating minor compaction (all files) 2024-12-05T05:18:07,028 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/B in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,028 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/A in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,028 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/aeb64cadd3864f5290f9ac0e37f08e48, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/fda179d558c642ab883743f97b9a17c7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/43b89764b7db456f80350d90521a772b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=100.4 K 2024-12-05T05:18:07,028 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ca9aecfcaaf04c98b8e1cf627d48c1d8, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/6ac919a03f3b4fc5a451860b8e53104a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f8772cabbeb446e0a993bb02ddad73fe] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=36.4 K 2024-12-05T05:18:07,028 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,028 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/aeb64cadd3864f5290f9ac0e37f08e48, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/fda179d558c642ab883743f97b9a17c7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/43b89764b7db456f80350d90521a772b] 2024-12-05T05:18:07,028 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ca9aecfcaaf04c98b8e1cf627d48c1d8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733375882491 2024-12-05T05:18:07,028 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb64cadd3864f5290f9ac0e37f08e48, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733375882491 2024-12-05T05:18:07,029 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting fda179d558c642ab883743f97b9a17c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375883191 2024-12-05T05:18:07,029 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ac919a03f3b4fc5a451860b8e53104a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375883191 2024-12-05T05:18:07,030 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43b89764b7db456f80350d90521a772b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733375885431 2024-12-05T05:18:07,030 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f8772cabbeb446e0a993bb02ddad73fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733375885431 2024-12-05T05:18:07,045 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:07,059 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#B#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:07,059 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/50e5fa11d4bf45519f4db478a5ef5355 is 50, key is test_row_0/B:col10/1733375885451/Put/seqid=0 2024-12-05T05:18:07,061 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205cb18cec7a7e94192af99faed9be27b5c_04aadddcc47c85ef20a018672c5865b6 store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:07,062 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205cb18cec7a7e94192af99faed9be27b5c_04aadddcc47c85ef20a018672c5865b6, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:07,063 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cb18cec7a7e94192af99faed9be27b5c_04aadddcc47c85ef20a018672c5865b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:07,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742252_1428 (size=13017) 2024-12-05T05:18:07,090 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/50e5fa11d4bf45519f4db478a5ef5355 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/50e5fa11d4bf45519f4db478a5ef5355 2024-12-05T05:18:07,095 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/B of 04aadddcc47c85ef20a018672c5865b6 into 50e5fa11d4bf45519f4db478a5ef5355(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:07,095 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:07,095 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/B, priority=13, startTime=1733375887026; duration=0sec 2024-12-05T05:18:07,095 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:07,095 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:B 2024-12-05T05:18:07,095 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:07,096 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:07,097 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/C is initiating minor compaction (all files) 2024-12-05T05:18:07,097 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/C in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,097 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6b0b51beb8bd4a5db673ef0f5d16706f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/99c4a00dea3544689cec46c0051ed32b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/73c7ccef230f44259f93a135c2629b5a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=36.4 K 2024-12-05T05:18:07,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742253_1429 (size=4469) 2024-12-05T05:18:07,097 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0b51beb8bd4a5db673ef0f5d16706f, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733375882491 2024-12-05T05:18:07,098 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 99c4a00dea3544689cec46c0051ed32b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733375883191 2024-12-05T05:18:07,098 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#A#compaction#357 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:07,099 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/680d352bdb0d4a49b7502ed6302baaf7 is 175, key is test_row_0/A:col10/1733375885451/Put/seqid=0 2024-12-05T05:18:07,099 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 73c7ccef230f44259f93a135c2629b5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733375885431 2024-12-05T05:18:07,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742254_1430 (size=31971) 2024-12-05T05:18:07,110 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/680d352bdb0d4a49b7502ed6302baaf7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/680d352bdb0d4a49b7502ed6302baaf7 2024-12-05T05:18:07,114 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/A of 04aadddcc47c85ef20a018672c5865b6 into 680d352bdb0d4a49b7502ed6302baaf7(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:07,115 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:07,115 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/A, priority=13, startTime=1733375887026; duration=0sec 2024-12-05T05:18:07,115 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:07,115 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:A 2024-12-05T05:18:07,115 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#C#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:07,115 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/17a23faffd3c443bbbad795c0052274e is 50, key is test_row_0/C:col10/1733375885451/Put/seqid=0 2024-12-05T05:18:07,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742255_1431 (size=13017) 2024-12-05T05:18:07,128 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/17a23faffd3c443bbbad795c0052274e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/17a23faffd3c443bbbad795c0052274e 2024-12-05T05:18:07,135 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/C of 04aadddcc47c85ef20a018672c5865b6 into 17a23faffd3c443bbbad795c0052274e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:07,135 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:07,135 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/C, priority=13, startTime=1733375887026; duration=0sec 2024-12-05T05:18:07,135 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:07,135 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:C 2024-12-05T05:18:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-05T05:18:07,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:07,317 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-05T05:18:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T05:18:07,320 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:07,321 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:07,321 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:07,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:18:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:07,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:07,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:07,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:07,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:07,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:07,324 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:07,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205193799fa3ac244c794b49ceccc9590a5_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375886147/Put/seqid=0 2024-12-05T05:18:07,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742256_1432 (size=14994) 2024-12-05T05:18:07,358 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:07,365 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205193799fa3ac244c794b49ceccc9590a5_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205193799fa3ac244c794b49ceccc9590a5_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:07,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:07,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e is 175, key is test_row_0/A:col10/1733375886147/Put/seqid=0 2024-12-05T05:18:07,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742257_1433 (size=39949) 2024-12-05T05:18:07,374 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e 2024-12-05T05:18:07,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/ddf56f03bcbe4fdda30ed694663bbc9f is 50, key is test_row_0/B:col10/1733375886147/Put/seqid=0 2024-12-05T05:18:07,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375947382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375947386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742258_1434 (size=12301) 2024-12-05T05:18:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T05:18:07,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T05:18:07,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:07,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375947492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375947503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T05:18:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T05:18:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37286 deadline: 1733375947642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,647 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37300 deadline: 1733375947672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,676 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375947713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37316 deadline: 1733375947723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375947723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:07,734 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in 04aadddcc47c85ef20a018672c5865b6 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18330 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:07,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T05:18:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/ddf56f03bcbe4fdda30ed694663bbc9f 2024-12-05T05:18:07,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c8478b6cb019418881b71b466a9b839f is 50, key is test_row_0/C:col10/1733375886147/Put/seqid=0 2024-12-05T05:18:07,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742259_1435 (size=12301) 2024-12-05T05:18:07,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T05:18:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:07,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:07,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T05:18:08,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:08,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375948022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:08,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375948043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:08,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T05:18:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:08,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:08,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:08,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:08,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:08,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c8478b6cb019418881b71b466a9b839f 2024-12-05T05:18:08,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T05:18:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:08,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:08,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:08,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e 2024-12-05T05:18:08,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e, entries=200, sequenceid=298, filesize=39.0 K 2024-12-05T05:18:08,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/ddf56f03bcbe4fdda30ed694663bbc9f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ddf56f03bcbe4fdda30ed694663bbc9f 2024-12-05T05:18:08,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ddf56f03bcbe4fdda30ed694663bbc9f, entries=150, sequenceid=298, filesize=12.0 K 2024-12-05T05:18:08,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/c8478b6cb019418881b71b466a9b839f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8478b6cb019418881b71b466a9b839f 2024-12-05T05:18:08,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8478b6cb019418881b71b466a9b839f, entries=150, sequenceid=298, filesize=12.0 K 2024-12-05T05:18:08,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 04aadddcc47c85ef20a018672c5865b6 in 958ms, sequenceid=298, compaction requested=false 2024-12-05T05:18:08,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:08,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-05T05:18:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:08,410 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T05:18:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:08,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:08,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cacbdc84bb80453c8464ac8605ba1b55_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375887380/Put/seqid=0 2024-12-05T05:18:08,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742260_1436 (size=12454) 2024-12-05T05:18:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T05:18:08,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:08,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. as already flushing 2024-12-05T05:18:08,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:08,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375948584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,587 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:08,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375948585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:08,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:08,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375948693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:08,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:08,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375948703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:08,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:08,831 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205cacbdc84bb80453c8464ac8605ba1b55_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cacbdc84bb80453c8464ac8605ba1b55_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/4acac1c461f84af0a4120a0e2d8c5382, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/4acac1c461f84af0a4120a0e2d8c5382 is 175, key is test_row_0/A:col10/1733375887380/Put/seqid=0 2024-12-05T05:18:08,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742261_1437 (size=31255) 2024-12-05T05:18:08,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375948903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:08,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375948922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:09,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:09,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375949213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:09,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:09,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375949232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:09,239 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:09,244 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/4acac1c461f84af0a4120a0e2d8c5382 2024-12-05T05:18:09,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/02df788e43b8485da347cd6bd6ae17fb is 50, key is test_row_0/B:col10/1733375887380/Put/seqid=0 2024-12-05T05:18:09,262 INFO [Thread-1580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1580. 2024-12-05T05:18:09,262 DEBUG [Thread-1580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:09,262 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,262 DEBUG [Thread-1580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:09,265 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-12-05T05:18:09,265 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:09,265 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,265 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,265 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:09,268 INFO [Thread-1576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1576. 2024-12-05T05:18:09,268 DEBUG [Thread-1576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:09,268 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,268 DEBUG [Thread-1576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,270 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:09,272 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-12-05T05:18:09,272 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:09,272 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,272 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:09,273 INFO [Thread-1578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1578. 2024-12-05T05:18:09,273 DEBUG [Thread-1578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:09,273 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,273 DEBUG [Thread-1578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:09,273 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:09,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742262_1438 (size=12301) 2024-12-05T05:18:09,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T05:18:09,518 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T05:18:09,674 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/02df788e43b8485da347cd6bd6ae17fb 2024-12-05T05:18:09,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/28fb8f6686c14f25bdb98f826c0aa12b is 50, key is test_row_0/C:col10/1733375887380/Put/seqid=0 2024-12-05T05:18:09,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742263_1439 (size=12301) 2024-12-05T05:18:09,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37350 deadline: 1733375949733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:09,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:09,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37284 deadline: 1733375949753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:09,754 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:10,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/28fb8f6686c14f25bdb98f826c0aa12b 2024-12-05T05:18:10,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/4acac1c461f84af0a4120a0e2d8c5382 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4acac1c461f84af0a4120a0e2d8c5382 2024-12-05T05:18:10,095 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4acac1c461f84af0a4120a0e2d8c5382, entries=150, sequenceid=316, filesize=30.5 K 2024-12-05T05:18:10,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/02df788e43b8485da347cd6bd6ae17fb as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/02df788e43b8485da347cd6bd6ae17fb 2024-12-05T05:18:10,098 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/02df788e43b8485da347cd6bd6ae17fb, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T05:18:10,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/28fb8f6686c14f25bdb98f826c0aa12b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/28fb8f6686c14f25bdb98f826c0aa12b 2024-12-05T05:18:10,101 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/28fb8f6686c14f25bdb98f826c0aa12b, entries=150, sequenceid=316, filesize=12.0 K 2024-12-05T05:18:10,102 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 04aadddcc47c85ef20a018672c5865b6 in 1692ms, sequenceid=316, compaction requested=true 2024-12-05T05:18:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-05T05:18:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-05T05:18:10,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-05T05:18:10,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7820 sec 2024-12-05T05:18:10,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.7870 sec 2024-12-05T05:18:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:10,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T05:18:10,744 INFO [Thread-1565 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1565. 2024-12-05T05:18:10,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:10,744 DEBUG [Thread-1565 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:10,744 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:10,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:10,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:10,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:10,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:10,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:10,744 DEBUG [Thread-1565 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:10,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:10,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ad3c27de83d3426aad3768e37084a39c_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375890743/Put/seqid=0 2024-12-05T05:18:10,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742264_1440 (size=12454) 2024-12-05T05:18:10,764 INFO [Thread-1571 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1571. 2024-12-05T05:18:10,764 DEBUG [Thread-1571 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:10,764 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:10,764 DEBUG [Thread-1571 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:10,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:11,151 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:11,154 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ad3c27de83d3426aad3768e37084a39c_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ad3c27de83d3426aad3768e37084a39c_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:11,154 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7fc906020377414c89b13e318a76c309, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:11,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7fc906020377414c89b13e318a76c309 is 175, key is test_row_0/A:col10/1733375890743/Put/seqid=0 2024-12-05T05:18:11,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742265_1441 (size=31255) 2024-12-05T05:18:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-05T05:18:11,504 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:11,559 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7fc906020377414c89b13e318a76c309 2024-12-05T05:18:11,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/2c74323b9e494a12bdaaa42b8634f049 is 50, key is test_row_0/B:col10/1733375890743/Put/seqid=0 2024-12-05T05:18:11,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742266_1442 (size=12301) 2024-12-05T05:18:11,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/2c74323b9e494a12bdaaa42b8634f049 2024-12-05T05:18:11,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/fb1761b78881424f866834cc57fdcdb3 is 50, key is test_row_0/C:col10/1733375890743/Put/seqid=0 2024-12-05T05:18:11,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742267_1443 (size=12301) 2024-12-05T05:18:12,385 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/fb1761b78881424f866834cc57fdcdb3 2024-12-05T05:18:12,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/7fc906020377414c89b13e318a76c309 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7fc906020377414c89b13e318a76c309 2024-12-05T05:18:12,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7fc906020377414c89b13e318a76c309, entries=150, sequenceid=331, filesize=30.5 K 2024-12-05T05:18:12,395 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/2c74323b9e494a12bdaaa42b8634f049 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2c74323b9e494a12bdaaa42b8634f049 2024-12-05T05:18:12,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2c74323b9e494a12bdaaa42b8634f049, entries=150, sequenceid=331, filesize=12.0 K 2024-12-05T05:18:12,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/fb1761b78881424f866834cc57fdcdb3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/fb1761b78881424f866834cc57fdcdb3 2024-12-05T05:18:12,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/fb1761b78881424f866834cc57fdcdb3, entries=150, sequenceid=331, filesize=12.0 K 2024-12-05T05:18:12,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for 04aadddcc47c85ef20a018672c5865b6 in 1661ms, sequenceid=331, compaction requested=true 2024-12-05T05:18:12,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:12,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:12,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:12,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:12,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:12,404 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:12,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 04aadddcc47c85ef20a018672c5865b6:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:12,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:12,404 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:12,405 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:12,405 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:12,406 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/B is initiating minor compaction (all files) 2024-12-05T05:18:12,406 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/A is initiating minor compaction (all files) 2024-12-05T05:18:12,406 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/A in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:12,406 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/B in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:12,406 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/680d352bdb0d4a49b7502ed6302baaf7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4acac1c461f84af0a4120a0e2d8c5382, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7fc906020377414c89b13e318a76c309] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=131.3 K 2024-12-05T05:18:12,406 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/50e5fa11d4bf45519f4db478a5ef5355, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ddf56f03bcbe4fdda30ed694663bbc9f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/02df788e43b8485da347cd6bd6ae17fb, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2c74323b9e494a12bdaaa42b8634f049] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=48.8 K 2024-12-05T05:18:12,406 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:12,406 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/680d352bdb0d4a49b7502ed6302baaf7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4acac1c461f84af0a4120a0e2d8c5382, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7fc906020377414c89b13e318a76c309] 2024-12-05T05:18:12,406 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 50e5fa11d4bf45519f4db478a5ef5355, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733375885431 2024-12-05T05:18:12,406 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 680d352bdb0d4a49b7502ed6302baaf7, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733375885431 2024-12-05T05:18:12,406 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ddf56f03bcbe4fdda30ed694663bbc9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733375886139 2024-12-05T05:18:12,406 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6ac52f0d8d64354ae6b5d9d9ae4a93e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733375886139 2024-12-05T05:18:12,407 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 02df788e43b8485da347cd6bd6ae17fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375887367 2024-12-05T05:18:12,407 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4acac1c461f84af0a4120a0e2d8c5382, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375887367 2024-12-05T05:18:12,407 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c74323b9e494a12bdaaa42b8634f049, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733375888583 2024-12-05T05:18:12,407 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fc906020377414c89b13e318a76c309, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733375888583 2024-12-05T05:18:12,416 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#B#compaction#369 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:12,416 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/d5d7825626d742baaf73615a2c993126 is 50, key is test_row_0/B:col10/1733375890743/Put/seqid=0 2024-12-05T05:18:12,416 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:12,418 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205361445a2d7b64175b45bff83cd1ddc79_04aadddcc47c85ef20a018672c5865b6 store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:12,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742268_1444 (size=13153) 2024-12-05T05:18:12,448 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205361445a2d7b64175b45bff83cd1ddc79_04aadddcc47c85ef20a018672c5865b6, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:12,448 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205361445a2d7b64175b45bff83cd1ddc79_04aadddcc47c85ef20a018672c5865b6 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:12,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742269_1445 (size=4469) 2024-12-05T05:18:12,829 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/d5d7825626d742baaf73615a2c993126 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d5d7825626d742baaf73615a2c993126 2024-12-05T05:18:12,833 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/B of 04aadddcc47c85ef20a018672c5865b6 into d5d7825626d742baaf73615a2c993126(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:12,833 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:12,833 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/B, priority=12, startTime=1733375892404; duration=0sec 2024-12-05T05:18:12,833 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:12,833 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:B 2024-12-05T05:18:12,833 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:12,835 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:12,835 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 04aadddcc47c85ef20a018672c5865b6/C is initiating minor compaction (all files) 2024-12-05T05:18:12,835 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 04aadddcc47c85ef20a018672c5865b6/C in TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:12,835 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/17a23faffd3c443bbbad795c0052274e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8478b6cb019418881b71b466a9b839f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/28fb8f6686c14f25bdb98f826c0aa12b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/fb1761b78881424f866834cc57fdcdb3] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp, totalSize=48.8 K 2024-12-05T05:18:12,835 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 17a23faffd3c443bbbad795c0052274e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1733375885431 2024-12-05T05:18:12,836 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c8478b6cb019418881b71b466a9b839f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1733375886139 2024-12-05T05:18:12,836 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 28fb8f6686c14f25bdb98f826c0aa12b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1733375887367 2024-12-05T05:18:12,836 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting fb1761b78881424f866834cc57fdcdb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1733375888583 2024-12-05T05:18:12,844 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#C#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:12,845 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/d07e1c000fa94bc2a033bffc5e5a1089 is 50, key is test_row_0/C:col10/1733375890743/Put/seqid=0 2024-12-05T05:18:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742270_1446 (size=13153) 2024-12-05T05:18:12,858 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 04aadddcc47c85ef20a018672c5865b6#A#compaction#370 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:12,859 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/9f0e5f7ec2d64ba6bed6589ed2f9029d is 175, key is test_row_0/A:col10/1733375890743/Put/seqid=0 2024-12-05T05:18:12,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742271_1447 (size=32107) 2024-12-05T05:18:13,256 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/d07e1c000fa94bc2a033bffc5e5a1089 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/d07e1c000fa94bc2a033bffc5e5a1089 2024-12-05T05:18:13,261 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/C of 04aadddcc47c85ef20a018672c5865b6 into d07e1c000fa94bc2a033bffc5e5a1089(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:13,261 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:13,262 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/C, priority=12, startTime=1733375892404; duration=0sec 2024-12-05T05:18:13,262 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:13,262 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:C 2024-12-05T05:18:13,269 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/9f0e5f7ec2d64ba6bed6589ed2f9029d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/9f0e5f7ec2d64ba6bed6589ed2f9029d 2024-12-05T05:18:13,273 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 04aadddcc47c85ef20a018672c5865b6/A of 04aadddcc47c85ef20a018672c5865b6 into 9f0e5f7ec2d64ba6bed6589ed2f9029d(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:13,273 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 04aadddcc47c85ef20a018672c5865b6: 2024-12-05T05:18:13,273 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., storeName=04aadddcc47c85ef20a018672c5865b6/A, priority=12, startTime=1733375892404; duration=0sec 2024-12-05T05:18:13,273 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:13,273 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 04aadddcc47c85ef20a018672c5865b6:A 2024-12-05T05:18:14,247 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e2cb8b41b7a44564aa66b09578cde7dc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e7639264e8594eb9b154a20094b8af25, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/6c51ca5cf9bc4170925600cdaceba652, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/a55bcb7b9ac540edb7efc38a6100239b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4d88c25edc254ca5a7226b0d59b3b49d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/12450a7f6b404bb1802259e39d761e2a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90997718a0a946b387b45d179169b1b7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/69dc0f076c1641a4b81e52f4e6777da7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7b54b5dd43e244ceaa9496a94c80ddad, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/913f3b5dacbe48acaf20b3bef9f31151, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/48b8aa967f5940e0b1a30a457a660680, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/73312bf237e14f38ae5d1c278b2b228d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90e5cb133eaf48638b822d8b75626bad, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/f4d85087461e4f058c36d094b8dfc200, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/ed66bbe1cc5943369c71c787628fb71c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7f3134ad1aa345be8891aa5a43a51259, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/08f1a5f65f094dbd9973b800962a1452, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/3718e4edfe514600b3c76aa94285a6e9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/aeb64cadd3864f5290f9ac0e37f08e48, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/fda179d558c642ab883743f97b9a17c7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/43b89764b7db456f80350d90521a772b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/680d352bdb0d4a49b7502ed6302baaf7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4acac1c461f84af0a4120a0e2d8c5382, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7fc906020377414c89b13e318a76c309] to archive 2024-12-05T05:18:14,248 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:18:14,250 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e2cb8b41b7a44564aa66b09578cde7dc to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e2cb8b41b7a44564aa66b09578cde7dc 2024-12-05T05:18:14,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e7639264e8594eb9b154a20094b8af25 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e7639264e8594eb9b154a20094b8af25 2024-12-05T05:18:14,251 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/6c51ca5cf9bc4170925600cdaceba652 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/6c51ca5cf9bc4170925600cdaceba652 2024-12-05T05:18:14,252 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/a55bcb7b9ac540edb7efc38a6100239b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/a55bcb7b9ac540edb7efc38a6100239b 2024-12-05T05:18:14,253 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4d88c25edc254ca5a7226b0d59b3b49d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4d88c25edc254ca5a7226b0d59b3b49d 2024-12-05T05:18:14,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/12450a7f6b404bb1802259e39d761e2a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/12450a7f6b404bb1802259e39d761e2a 2024-12-05T05:18:14,254 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90997718a0a946b387b45d179169b1b7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90997718a0a946b387b45d179169b1b7 2024-12-05T05:18:14,255 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/69dc0f076c1641a4b81e52f4e6777da7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/69dc0f076c1641a4b81e52f4e6777da7 2024-12-05T05:18:14,256 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7b54b5dd43e244ceaa9496a94c80ddad to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7b54b5dd43e244ceaa9496a94c80ddad 2024-12-05T05:18:14,257 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/913f3b5dacbe48acaf20b3bef9f31151 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/913f3b5dacbe48acaf20b3bef9f31151 2024-12-05T05:18:14,258 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/48b8aa967f5940e0b1a30a457a660680 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/48b8aa967f5940e0b1a30a457a660680 2024-12-05T05:18:14,259 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/73312bf237e14f38ae5d1c278b2b228d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/73312bf237e14f38ae5d1c278b2b228d 2024-12-05T05:18:14,261 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90e5cb133eaf48638b822d8b75626bad to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/90e5cb133eaf48638b822d8b75626bad 2024-12-05T05:18:14,262 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/f4d85087461e4f058c36d094b8dfc200 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/f4d85087461e4f058c36d094b8dfc200 2024-12-05T05:18:14,263 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/ed66bbe1cc5943369c71c787628fb71c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/ed66bbe1cc5943369c71c787628fb71c 2024-12-05T05:18:14,265 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7f3134ad1aa345be8891aa5a43a51259 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7f3134ad1aa345be8891aa5a43a51259 2024-12-05T05:18:14,266 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/08f1a5f65f094dbd9973b800962a1452 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/08f1a5f65f094dbd9973b800962a1452 2024-12-05T05:18:14,268 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/3718e4edfe514600b3c76aa94285a6e9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/3718e4edfe514600b3c76aa94285a6e9 2024-12-05T05:18:14,269 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/aeb64cadd3864f5290f9ac0e37f08e48 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/aeb64cadd3864f5290f9ac0e37f08e48 2024-12-05T05:18:14,270 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/fda179d558c642ab883743f97b9a17c7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/fda179d558c642ab883743f97b9a17c7 2024-12-05T05:18:14,272 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/43b89764b7db456f80350d90521a772b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/43b89764b7db456f80350d90521a772b 2024-12-05T05:18:14,273 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/680d352bdb0d4a49b7502ed6302baaf7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/680d352bdb0d4a49b7502ed6302baaf7 2024-12-05T05:18:14,274 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/e6ac52f0d8d64354ae6b5d9d9ae4a93e 2024-12-05T05:18:14,275 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4acac1c461f84af0a4120a0e2d8c5382 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/4acac1c461f84af0a4120a0e2d8c5382 2024-12-05T05:18:14,276 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7fc906020377414c89b13e318a76c309 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/7fc906020377414c89b13e318a76c309 2024-12-05T05:18:14,280 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/1af21d16421e4605bd006e653765b620, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d0dd75a50e3a4268b40bb81ba32f15e2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f05efcc37df54f5a8573ddb342e99dd3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/109cfe1a56164be0ac17f6878e5d961f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/583b719f0db744ad8e8c45d930d49d87, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0014a34c84c9449aac0ff9eff4f5cc54, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ffe9686a3b6943e898b3200460e91424, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/304fb357bf59415984bf66c54a24f7e6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/21733c6a1fc946e295aa57ce702448bc, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0cbd6af9e7d64f0595abb14e59f4c9e1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/cc9f3275a151465d91b282868c209a21, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/a2c803b08eae498a89420fc894342245, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/62483fc4d9e9467a91e0d9c1cd1f4f08, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2ba4420114694c5e8da6f46123c791a3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/c7da41bf11b849d296cae815ee352f0e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/b064056162fd4a6a93a3c7c215eff80f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ca9aecfcaaf04c98b8e1cf627d48c1d8, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/3fe8fba0d93a49d18349909cdc857436, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/6ac919a03f3b4fc5a451860b8e53104a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/50e5fa11d4bf45519f4db478a5ef5355, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f8772cabbeb446e0a993bb02ddad73fe, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ddf56f03bcbe4fdda30ed694663bbc9f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/02df788e43b8485da347cd6bd6ae17fb, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2c74323b9e494a12bdaaa42b8634f049] to archive 2024-12-05T05:18:14,282 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:18:14,284 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/1af21d16421e4605bd006e653765b620 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/1af21d16421e4605bd006e653765b620 2024-12-05T05:18:14,286 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d0dd75a50e3a4268b40bb81ba32f15e2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d0dd75a50e3a4268b40bb81ba32f15e2 2024-12-05T05:18:14,287 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f05efcc37df54f5a8573ddb342e99dd3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f05efcc37df54f5a8573ddb342e99dd3 2024-12-05T05:18:14,288 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/109cfe1a56164be0ac17f6878e5d961f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/109cfe1a56164be0ac17f6878e5d961f 2024-12-05T05:18:14,289 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/583b719f0db744ad8e8c45d930d49d87 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/583b719f0db744ad8e8c45d930d49d87 2024-12-05T05:18:14,290 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0014a34c84c9449aac0ff9eff4f5cc54 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0014a34c84c9449aac0ff9eff4f5cc54 2024-12-05T05:18:14,291 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ffe9686a3b6943e898b3200460e91424 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ffe9686a3b6943e898b3200460e91424 2024-12-05T05:18:14,294 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/304fb357bf59415984bf66c54a24f7e6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/304fb357bf59415984bf66c54a24f7e6 2024-12-05T05:18:14,295 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/21733c6a1fc946e295aa57ce702448bc to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/21733c6a1fc946e295aa57ce702448bc 2024-12-05T05:18:14,296 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/dc1d4e4424ba4cb3b2e84a07aefa6cd3 2024-12-05T05:18:14,299 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0cbd6af9e7d64f0595abb14e59f4c9e1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/0cbd6af9e7d64f0595abb14e59f4c9e1 2024-12-05T05:18:14,300 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/cc9f3275a151465d91b282868c209a21 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/cc9f3275a151465d91b282868c209a21 2024-12-05T05:18:14,308 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/a2c803b08eae498a89420fc894342245 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/a2c803b08eae498a89420fc894342245 2024-12-05T05:18:14,309 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/62483fc4d9e9467a91e0d9c1cd1f4f08 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/62483fc4d9e9467a91e0d9c1cd1f4f08 2024-12-05T05:18:14,310 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2ba4420114694c5e8da6f46123c791a3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2ba4420114694c5e8da6f46123c791a3 2024-12-05T05:18:14,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/c7da41bf11b849d296cae815ee352f0e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/c7da41bf11b849d296cae815ee352f0e 2024-12-05T05:18:14,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/b064056162fd4a6a93a3c7c215eff80f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/b064056162fd4a6a93a3c7c215eff80f 2024-12-05T05:18:14,312 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ca9aecfcaaf04c98b8e1cf627d48c1d8 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ca9aecfcaaf04c98b8e1cf627d48c1d8 2024-12-05T05:18:14,313 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/3fe8fba0d93a49d18349909cdc857436 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/3fe8fba0d93a49d18349909cdc857436 2024-12-05T05:18:14,314 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/6ac919a03f3b4fc5a451860b8e53104a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/6ac919a03f3b4fc5a451860b8e53104a 2024-12-05T05:18:14,314 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/50e5fa11d4bf45519f4db478a5ef5355 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/50e5fa11d4bf45519f4db478a5ef5355 2024-12-05T05:18:14,315 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f8772cabbeb446e0a993bb02ddad73fe to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/f8772cabbeb446e0a993bb02ddad73fe 2024-12-05T05:18:14,316 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ddf56f03bcbe4fdda30ed694663bbc9f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/ddf56f03bcbe4fdda30ed694663bbc9f 2024-12-05T05:18:14,316 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/02df788e43b8485da347cd6bd6ae17fb to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/02df788e43b8485da347cd6bd6ae17fb 2024-12-05T05:18:14,317 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2c74323b9e494a12bdaaa42b8634f049 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/2c74323b9e494a12bdaaa42b8634f049 2024-12-05T05:18:14,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/54d43dd89bcd4709a07ba1d1c17f802d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/26a39eb5cd3c41f88196611ba686a01b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/283cd3a453144252bf4fe0885233b6d2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/7bfc6dec89dc4226a4633e9b77de2a85, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/56671c08e6194475839232c7a8298d32, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6274a27ea5eb48c8868178a64a3f00e3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5d4282d70451428fb41abc51fa4dab7f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8bfe4437ef647f49aa40fa5c1d97f4a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/b944f8ecf3f34cc1a0b3a7837be333a9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/46e25febf03d4687b94d95cdd2826a58, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c08e10b79e64459d858914ed0dabfe6f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/10eb269e5a9e4707a02cf83d874b8868, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/157b292513414743a469f2cab9fe4ee8, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3a0586ec7a4e472294f9c92b68c950ab, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3f8bf4ba40e14625acb2b3192e7218f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/ab8adefc39f749089da13b5b2cb48572, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/a359bbc597354b68a9019ae97c56fc43, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6b0b51beb8bd4a5db673ef0f5d16706f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/0adb547e50d7437088881ecaf4b46401, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/99c4a00dea3544689cec46c0051ed32b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/17a23faffd3c443bbbad795c0052274e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/73c7ccef230f44259f93a135c2629b5a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8478b6cb019418881b71b466a9b839f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/28fb8f6686c14f25bdb98f826c0aa12b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/fb1761b78881424f866834cc57fdcdb3] to archive 2024-12-05T05:18:14,322 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:18:14,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/54d43dd89bcd4709a07ba1d1c17f802d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/54d43dd89bcd4709a07ba1d1c17f802d 2024-12-05T05:18:14,324 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/26a39eb5cd3c41f88196611ba686a01b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/26a39eb5cd3c41f88196611ba686a01b 2024-12-05T05:18:14,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/283cd3a453144252bf4fe0885233b6d2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/283cd3a453144252bf4fe0885233b6d2 2024-12-05T05:18:14,327 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/7bfc6dec89dc4226a4633e9b77de2a85 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/7bfc6dec89dc4226a4633e9b77de2a85 2024-12-05T05:18:14,328 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/56671c08e6194475839232c7a8298d32 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/56671c08e6194475839232c7a8298d32 2024-12-05T05:18:14,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6274a27ea5eb48c8868178a64a3f00e3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6274a27ea5eb48c8868178a64a3f00e3 2024-12-05T05:18:14,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5d4282d70451428fb41abc51fa4dab7f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5d4282d70451428fb41abc51fa4dab7f 2024-12-05T05:18:14,330 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8bfe4437ef647f49aa40fa5c1d97f4a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8bfe4437ef647f49aa40fa5c1d97f4a 2024-12-05T05:18:14,331 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/b944f8ecf3f34cc1a0b3a7837be333a9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/b944f8ecf3f34cc1a0b3a7837be333a9 2024-12-05T05:18:14,332 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/46e25febf03d4687b94d95cdd2826a58 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/46e25febf03d4687b94d95cdd2826a58 2024-12-05T05:18:14,334 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c08e10b79e64459d858914ed0dabfe6f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c08e10b79e64459d858914ed0dabfe6f 2024-12-05T05:18:14,335 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/10eb269e5a9e4707a02cf83d874b8868 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/10eb269e5a9e4707a02cf83d874b8868 2024-12-05T05:18:14,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/157b292513414743a469f2cab9fe4ee8 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/157b292513414743a469f2cab9fe4ee8 2024-12-05T05:18:14,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3a0586ec7a4e472294f9c92b68c950ab to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3a0586ec7a4e472294f9c92b68c950ab 2024-12-05T05:18:14,338 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3f8bf4ba40e14625acb2b3192e7218f5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/3f8bf4ba40e14625acb2b3192e7218f5 2024-12-05T05:18:14,339 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/ab8adefc39f749089da13b5b2cb48572 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/ab8adefc39f749089da13b5b2cb48572 2024-12-05T05:18:14,341 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/a359bbc597354b68a9019ae97c56fc43 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/a359bbc597354b68a9019ae97c56fc43 2024-12-05T05:18:14,342 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6b0b51beb8bd4a5db673ef0f5d16706f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/6b0b51beb8bd4a5db673ef0f5d16706f 2024-12-05T05:18:14,343 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/0adb547e50d7437088881ecaf4b46401 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/0adb547e50d7437088881ecaf4b46401 2024-12-05T05:18:14,344 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/99c4a00dea3544689cec46c0051ed32b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/99c4a00dea3544689cec46c0051ed32b 2024-12-05T05:18:14,345 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/17a23faffd3c443bbbad795c0052274e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/17a23faffd3c443bbbad795c0052274e 2024-12-05T05:18:14,346 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/73c7ccef230f44259f93a135c2629b5a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/73c7ccef230f44259f93a135c2629b5a 2024-12-05T05:18:14,347 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8478b6cb019418881b71b466a9b839f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/c8478b6cb019418881b71b466a9b839f 2024-12-05T05:18:14,348 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/28fb8f6686c14f25bdb98f826c0aa12b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/28fb8f6686c14f25bdb98f826c0aa12b 2024-12-05T05:18:14,348 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/8edcaf2ecc84:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/fb1761b78881424f866834cc57fdcdb3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/fb1761b78881424f866834cc57fdcdb3 2024-12-05T05:18:17,744 INFO [Thread-1569 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1569. 2024-12-05T05:18:17,744 DEBUG [Thread-1569 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:17,744 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:17,744 DEBUG [Thread-1569 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:17,744 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:17,754 INFO [Thread-1567 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1567. 2024-12-05T05:18:17,754 INFO [Thread-1573 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1573. 2024-12-05T05:18:17,754 DEBUG [Thread-1573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:17,754 DEBUG [Thread-1567 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:17,754 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:17,754 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:17,754 DEBUG [Thread-1573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:17,754 DEBUG [Thread-1567 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:17,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:17,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1335 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4004 rows 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1337 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4008 rows 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1322 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3966 rows 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1319 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3957 rows 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1332 2024-12-05T05:18:17,755 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3995 rows 2024-12-05T05:18:17,755 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T05:18:17,755 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:18:17,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:17,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:17,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:17,756 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T05:18:17,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:17,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T05:18:17,759 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375897759"}]},"ts":"1733375897759"} 2024-12-05T05:18:17,760 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T05:18:17,760 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T05:18:17,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:18:17,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, UNASSIGN}] 2024-12-05T05:18:17,763 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, UNASSIGN 2024-12-05T05:18:17,764 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=04aadddcc47c85ef20a018672c5865b6, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:17,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, UNASSIGN because future has completed 2024-12-05T05:18:17,765 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:18:17,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:18:17,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T05:18:17,916 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:17,916 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing 04aadddcc47c85ef20a018672c5865b6, disabling compactions & flushes 2024-12-05T05:18:17,917 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. after waiting 0 ms 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:17,917 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing 04aadddcc47c85ef20a018672c5865b6 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=A 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=B 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 04aadddcc47c85ef20a018672c5865b6, store=C 2024-12-05T05:18:17,917 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:17,922 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059e800c7ea65b4fe8b925263e8df7946d_04aadddcc47c85ef20a018672c5865b6 is 50, key is test_row_0/A:col10/1733375897753/Put/seqid=0 2024-12-05T05:18:17,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742272_1448 (size=9914) 2024-12-05T05:18:18,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T05:18:18,327 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:18,331 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059e800c7ea65b4fe8b925263e8df7946d_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059e800c7ea65b4fe8b925263e8df7946d_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:18,332 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/180f33c7d1f443e996237c5f1a39f607, store: [table=TestAcidGuarantees family=A region=04aadddcc47c85ef20a018672c5865b6] 2024-12-05T05:18:18,332 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/180f33c7d1f443e996237c5f1a39f607 is 175, key is test_row_0/A:col10/1733375897753/Put/seqid=0 2024-12-05T05:18:18,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742273_1449 (size=22561) 2024-12-05T05:18:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T05:18:18,738 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=341, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/180f33c7d1f443e996237c5f1a39f607 2024-12-05T05:18:18,750 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/bde5a421d81c492ab477fa82a2e0b510 is 50, key is test_row_0/B:col10/1733375897753/Put/seqid=0 2024-12-05T05:18:18,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742274_1450 (size=9857) 2024-12-05T05:18:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T05:18:19,154 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/bde5a421d81c492ab477fa82a2e0b510 2024-12-05T05:18:19,160 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/5ed860a330214c788c4acade9c93a8dd is 50, key is test_row_0/C:col10/1733375897753/Put/seqid=0 2024-12-05T05:18:19,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742275_1451 (size=9857) 2024-12-05T05:18:19,564 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/5ed860a330214c788c4acade9c93a8dd 2024-12-05T05:18:19,568 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/A/180f33c7d1f443e996237c5f1a39f607 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/180f33c7d1f443e996237c5f1a39f607 2024-12-05T05:18:19,573 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/180f33c7d1f443e996237c5f1a39f607, entries=100, sequenceid=341, filesize=22.0 K 2024-12-05T05:18:19,574 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/B/bde5a421d81c492ab477fa82a2e0b510 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/bde5a421d81c492ab477fa82a2e0b510 2024-12-05T05:18:19,580 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/bde5a421d81c492ab477fa82a2e0b510, entries=100, sequenceid=341, filesize=9.6 K 2024-12-05T05:18:19,581 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/.tmp/C/5ed860a330214c788c4acade9c93a8dd as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5ed860a330214c788c4acade9c93a8dd 2024-12-05T05:18:19,586 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5ed860a330214c788c4acade9c93a8dd, entries=100, sequenceid=341, filesize=9.6 K 2024-12-05T05:18:19,587 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 04aadddcc47c85ef20a018672c5865b6 in 1670ms, sequenceid=341, compaction requested=false 2024-12-05T05:18:19,591 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/recovered.edits/344.seqid, newMaxSeqId=344, maxSeqId=4 2024-12-05T05:18:19,592 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. 2024-12-05T05:18:19,592 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for 04aadddcc47c85ef20a018672c5865b6: Waiting for close lock at 1733375897917Running coprocessor pre-close hooks at 1733375897917Disabling compacts and flushes for region at 1733375897917Disabling writes for close at 1733375897917Obtaining lock to block concurrent updates at 1733375897917Preparing flush snapshotting stores in 04aadddcc47c85ef20a018672c5865b6 at 1733375897917Finished memstore snapshotting TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733375897917Flushing stores of TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6. at 1733375897918 (+1 ms)Flushing 04aadddcc47c85ef20a018672c5865b6/A: creating writer at 1733375897918Flushing mob file 04aadddcc47c85ef20a018672c5865b6/A: appending metadata at 1733375897922 (+4 ms)Flushing mob file 04aadddcc47c85ef20a018672c5865b6/A: closing flushed file at 1733375897922Flushing 04aadddcc47c85ef20a018672c5865b6/A: appending metadata at 1733375898332 (+410 ms)Flushing 04aadddcc47c85ef20a018672c5865b6/A: closing flushed file at 1733375898332Flushing 04aadddcc47c85ef20a018672c5865b6/B: creating writer at 1733375898747 (+415 ms)Flushing 04aadddcc47c85ef20a018672c5865b6/B: appending metadata at 1733375898750 (+3 ms)Flushing 04aadddcc47c85ef20a018672c5865b6/B: closing flushed file at 1733375898750Flushing 04aadddcc47c85ef20a018672c5865b6/C: creating writer at 1733375899157 (+407 ms)Flushing 04aadddcc47c85ef20a018672c5865b6/C: appending metadata at 1733375899160 (+3 ms)Flushing 04aadddcc47c85ef20a018672c5865b6/C: closing flushed file at 1733375899160Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c73ea8f: reopening flushed file at 1733375899568 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78e7e488: reopening flushed file at 1733375899574 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@279f4662: reopening flushed file at 1733375899580 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 04aadddcc47c85ef20a018672c5865b6 in 1670ms, sequenceid=341, compaction requested=false at 1733375899587 (+7 ms)Writing region close event to WAL at 1733375899588 (+1 ms)Running coprocessor post-close hooks at 1733375899592 (+4 ms)Closed at 1733375899592 2024-12-05T05:18:19,594 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed 04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,595 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=04aadddcc47c85ef20a018672c5865b6, regionState=CLOSED 2024-12-05T05:18:19,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:18:19,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-05T05:18:19,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure 04aadddcc47c85ef20a018672c5865b6, server=8edcaf2ecc84,36233,1733375771401 in 1.8320 sec 2024-12-05T05:18:19,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-05T05:18:19,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=04aadddcc47c85ef20a018672c5865b6, UNASSIGN in 1.8360 sec 2024-12-05T05:18:19,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-05T05:18:19,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8390 sec 2024-12-05T05:18:19,603 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375899603"}]},"ts":"1733375899603"} 2024-12-05T05:18:19,604 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T05:18:19,604 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T05:18:19,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8490 sec 2024-12-05T05:18:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-05T05:18:19,903 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:19,904 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T05:18:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:19,907 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:19,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T05:18:19,909 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:19,914 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,916 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/recovered.edits] 2024-12-05T05:18:19,918 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/180f33c7d1f443e996237c5f1a39f607 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/180f33c7d1f443e996237c5f1a39f607 2024-12-05T05:18:19,919 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/9f0e5f7ec2d64ba6bed6589ed2f9029d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/A/9f0e5f7ec2d64ba6bed6589ed2f9029d 2024-12-05T05:18:19,922 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/bde5a421d81c492ab477fa82a2e0b510 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/bde5a421d81c492ab477fa82a2e0b510 2024-12-05T05:18:19,923 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d5d7825626d742baaf73615a2c993126 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/B/d5d7825626d742baaf73615a2c993126 2024-12-05T05:18:19,925 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5ed860a330214c788c4acade9c93a8dd to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/5ed860a330214c788c4acade9c93a8dd 2024-12-05T05:18:19,926 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/d07e1c000fa94bc2a033bffc5e5a1089 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/C/d07e1c000fa94bc2a033bffc5e5a1089 2024-12-05T05:18:19,929 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/recovered.edits/344.seqid to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6/recovered.edits/344.seqid 2024-12-05T05:18:19,929 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,929 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T05:18:19,929 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T05:18:19,930 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T05:18:19,932 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050d3ae972bb5b4a78aec7084834f1e260_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050d3ae972bb5b4a78aec7084834f1e260_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,933 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205193799fa3ac244c794b49ceccc9590a5_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205193799fa3ac244c794b49ceccc9590a5_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,933 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053adc010760694e36b069922a67d91bcc_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053adc010760694e36b069922a67d91bcc_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,934 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053d13526732de4f01be37ce7cfbc606bc_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053d13526732de4f01be37ce7cfbc606bc_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,935 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120546fbae056c9d4e46bc723f68d7ff482e_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120546fbae056c9d4e46bc723f68d7ff482e_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,935 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120548c6d0f64c234127b003abefdb01703a_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120548c6d0f64c234127b003abefdb01703a_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,936 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055149b92cc1cb473dae8c6976bcd7722c_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055149b92cc1cb473dae8c6976bcd7722c_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,937 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ae2cf60e07c4c9f8e8dee1b9a699e13_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ae2cf60e07c4c9f8e8dee1b9a699e13_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,937 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120593b82a08275c4330bfc4d2567b2150da_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120593b82a08275c4330bfc4d2567b2150da_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,938 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120598d9551035a9487fbdaef34e51aad1c1_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120598d9551035a9487fbdaef34e51aad1c1_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,938 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120598ffaba893184c419641a6fbe5c520bb_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120598ffaba893184c419641a6fbe5c520bb_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,939 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b9ab83682694d7b85242d277ee7365e_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059b9ab83682694d7b85242d277ee7365e_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,940 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059d9019effde94ff9ab392f162704e048_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059d9019effde94ff9ab392f162704e048_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,941 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059e800c7ea65b4fe8b925263e8df7946d_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059e800c7ea65b4fe8b925263e8df7946d_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,941 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ad3c27de83d3426aad3768e37084a39c_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205ad3c27de83d3426aad3768e37084a39c_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,942 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b8738ca440df41ca89111ce3e0f42787_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205b8738ca440df41ca89111ce3e0f42787_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,943 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c517b48414804e41a289859eb8b02012_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c517b48414804e41a289859eb8b02012_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,943 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cacbdc84bb80453c8464ac8605ba1b55_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205cacbdc84bb80453c8464ac8605ba1b55_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,944 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eda682839aee4c23ab05165f0b217e2e_04aadddcc47c85ef20a018672c5865b6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eda682839aee4c23ab05165f0b217e2e_04aadddcc47c85ef20a018672c5865b6 2024-12-05T05:18:19,945 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T05:18:19,947 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:19,949 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T05:18:19,951 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T05:18:19,952 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:19,952 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T05:18:19,952 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733375899952"}]},"ts":"9223372036854775807"} 2024-12-05T05:18:19,954 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T05:18:19,954 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 04aadddcc47c85ef20a018672c5865b6, NAME => 'TestAcidGuarantees,,1733375866093.04aadddcc47c85ef20a018672c5865b6.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T05:18:19,954 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T05:18:19,955 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733375899954"}]},"ts":"9223372036854775807"} 2024-12-05T05:18:19,956 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T05:18:19,957 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:19,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-12-05T05:18:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-05T05:18:20,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T05:18:20,014 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:20,033 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 246) - Thread LEAK? -, OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=636 (was 616) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3378 (was 3320) - AvailableMemoryMB LEAK? - 2024-12-05T05:18:20,040 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=636, ProcessCount=11, AvailableMemoryMB=3378 2024-12-05T05:18:20,041 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:18:20,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:18:20,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:20,043 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T05:18:20,043 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:20,043 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-12-05T05:18:20,044 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T05:18:20,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-05T05:18:20,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742276_1452 (size=1061) 2024-12-05T05:18:20,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-05T05:18:20,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-05T05:18:20,451 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 2024-12-05T05:18:20,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742277_1453 (size=53) 2024-12-05T05:18:20,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-05T05:18:20,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:18:20,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 41b82809bb37e31af4591859446916bc, disabling compactions & flushes 2024-12-05T05:18:20,858 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:20,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:20,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. after waiting 0 ms 2024-12-05T05:18:20,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:20,858 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:20,858 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 41b82809bb37e31af4591859446916bc: Waiting for close lock at 1733375900858Disabling compacts and flushes for region at 1733375900858Disabling writes for close at 1733375900858Writing region close event to WAL at 1733375900858Closed at 1733375900858 2024-12-05T05:18:20,860 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T05:18:20,861 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733375900860"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733375900860"}]},"ts":"1733375900860"} 2024-12-05T05:18:20,863 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T05:18:20,864 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T05:18:20,864 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375900864"}]},"ts":"1733375900864"} 2024-12-05T05:18:20,866 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T05:18:20,867 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, ASSIGN}] 2024-12-05T05:18:20,868 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, ASSIGN 2024-12-05T05:18:20,869 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, ASSIGN; state=OFFLINE, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=false 2024-12-05T05:18:20,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:18:21,020 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=41b82809bb37e31af4591859446916bc, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:21,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, ASSIGN because future has completed 2024-12-05T05:18:21,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:18:21,179 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:21,179 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:18:21,182 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,182 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:18:21,182 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,182 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-05T05:18:21,185 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,187 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:21,187 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41b82809bb37e31af4591859446916bc columnFamilyName A 2024-12-05T05:18:21,187 DEBUG [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:21,188 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.HStore(327): Store=41b82809bb37e31af4591859446916bc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:21,188 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,189 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:21,189 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41b82809bb37e31af4591859446916bc columnFamilyName B 2024-12-05T05:18:21,189 DEBUG [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:21,190 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.HStore(327): Store=41b82809bb37e31af4591859446916bc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:21,190 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,191 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:21,191 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 41b82809bb37e31af4591859446916bc columnFamilyName C 2024-12-05T05:18:21,191 DEBUG [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:21,192 INFO [StoreOpener-41b82809bb37e31af4591859446916bc-1 {}] regionserver.HStore(327): Store=41b82809bb37e31af4591859446916bc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:21,192 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:21,192 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,193 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,193 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,194 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,194 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,194 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:18:21,195 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,197 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T05:18:21,197 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 41b82809bb37e31af4591859446916bc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61707408, jitterRate=-0.08048796653747559}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:18:21,197 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:21,197 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 41b82809bb37e31af4591859446916bc: Running coprocessor pre-open hook at 1733375901182Writing region info on filesystem at 1733375901182Initializing all the Stores at 1733375901184 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375901184Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375901184Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375901185 (+1 ms)Cleaning up temporary data from old regions at 1733375901194 (+9 ms)Running coprocessor post-open hooks at 1733375901197 (+3 ms)Region opened successfully at 1733375901197 2024-12-05T05:18:21,198 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., pid=112, masterSystemTime=1733375901174 2024-12-05T05:18:21,200 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:21,200 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:21,200 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=41b82809bb37e31af4591859446916bc, regionState=OPEN, openSeqNum=2, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:21,202 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:18:21,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-05T05:18:21,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 in 180 msec 2024-12-05T05:18:21,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-12-05T05:18:21,205 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, ASSIGN in 338 msec 2024-12-05T05:18:21,206 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T05:18:21,206 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375901206"}]},"ts":"1733375901206"} 2024-12-05T05:18:21,207 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T05:18:21,208 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T05:18:21,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-12-05T05:18:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-05T05:18:22,193 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:22,194 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@309953ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,194 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,194 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,195 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@714ff7a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,196 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,197 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,197 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44613f9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,198 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,198 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,200 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T05:18:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 8edcaf2ecc84,46695,1733375770505 2024-12-05T05:18:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c940b0e 2024-12-05T05:18:22,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T05:18:22,202 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T05:18:22,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135be63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,205 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf5dec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,206 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,206 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dfa651f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@432aef64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,209 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74012e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,209 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,210 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a6ab8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a177044, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,213 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7708a1fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,214 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,215 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47c8d48d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbbba3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,216 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@712a7b7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,218 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b78741b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cd9b86d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,220 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1fcfa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,222 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,223 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70207a68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48bc9769, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,224 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502bdfaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,225 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,226 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50220, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29cbd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@460a8c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,229 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,229 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@269bb9ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,231 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@731a2ab3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b76d744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,233 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,233 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2d8294, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,234 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,236 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,236 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55bdb25a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@694dc39c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,238 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3247f9e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,240 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e026bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f57cd4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,241 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:22,241 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:22,242 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a00f5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:22,242 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:22,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:22,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:22,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c32a6f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:22,244 DEBUG [Thread-2051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,245 DEBUG [Thread-2042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,245 DEBUG [Thread-2053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,245 DEBUG [Thread-2047 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,245 DEBUG [Thread-2038 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,245 DEBUG [Thread-2055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,245 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,246 DEBUG [Thread-2036 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,247 DEBUG [Thread-2044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,248 DEBUG [Thread-2040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,248 DEBUG [Thread-2049 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:22,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:22,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,249 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,250 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-12-05T05:18:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:22,251 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:22,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T05:18:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,253 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,253 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,254 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:22,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:22,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2] 2024-12-05T05:18:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:22,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:18:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:22,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:22,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/8a7530a0c1784220901ba434fe1a82a9 is 50, key is test_row_0/A:col10/1733375902256/Put/seqid=0 2024-12-05T05:18:22,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742278_1454 (size=14341) 2024-12-05T05:18:22,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375962309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375962311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375902308, max timestamp=1733375902318Num uniques 859; 2024-12-05T05:18:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375902308, max timestamp=1733375902318Num uniques 859; 2024-12-05T05:18:22,321 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375962311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375902308, max timestamp=1733375902318Num uniques 859; 2024-12-05T05:18:22,322 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T05:18:22,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375962371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,376 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375962376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T05:18:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:22,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375962423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,425 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375962432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375962433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,437 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375962482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375962483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T05:18:22,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:22,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:22,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:22,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T05:18:22,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375962633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,642 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375962643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375962643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375962693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375962693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T05:18:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:22,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:22,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/8a7530a0c1784220901ba434fe1a82a9 2024-12-05T05:18:22,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/7707a59ce9f447a993583fc7241d2f41 is 50, key is test_row_0/B:col10/1733375902256/Put/seqid=0 2024-12-05T05:18:22,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742279_1455 (size=12001) 2024-12-05T05:18:22,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/7707a59ce9f447a993583fc7241d2f41 2024-12-05T05:18:22,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/a19662f599f24d8d8e57fd41a5bfcd1c is 50, key is test_row_0/C:col10/1733375902256/Put/seqid=0 2024-12-05T05:18:22,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742280_1456 (size=12001) 2024-12-05T05:18:22,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/a19662f599f24d8d8e57fd41a5bfcd1c 2024-12-05T05:18:22,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/8a7530a0c1784220901ba434fe1a82a9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8a7530a0c1784220901ba434fe1a82a9 2024-12-05T05:18:22,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8a7530a0c1784220901ba434fe1a82a9, entries=200, sequenceid=12, filesize=14.0 K 2024-12-05T05:18:22,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/7707a59ce9f447a993583fc7241d2f41 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/7707a59ce9f447a993583fc7241d2f41 2024-12-05T05:18:22,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/7707a59ce9f447a993583fc7241d2f41, entries=150, sequenceid=12, filesize=11.7 K 2024-12-05T05:18:22,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/a19662f599f24d8d8e57fd41a5bfcd1c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a19662f599f24d8d8e57fd41a5bfcd1c 2024-12-05T05:18:22,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a19662f599f24d8d8e57fd41a5bfcd1c, entries=150, sequenceid=12, filesize=11.7 K 2024-12-05T05:18:22,803 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 41b82809bb37e31af4591859446916bc in 521ms, sequenceid=12, compaction requested=false 2024-12-05T05:18:22,803 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T05:18:22,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:22,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-12-05T05:18:22,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:22,866 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-05T05:18:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:22,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:22,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/83033ab833b140d4949e1a297553c852 is 50, key is test_row_0/A:col10/1733375902367/Put/seqid=0 2024-12-05T05:18:22,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T05:18:22,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742281_1457 (size=12001) 2024-12-05T05:18:22,917 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/83033ab833b140d4949e1a297553c852 2024-12-05T05:18:22,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/33fc6b27327542c4b0d2755a1876d2de is 50, key is test_row_0/B:col10/1733375902367/Put/seqid=0 2024-12-05T05:18:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:22,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:22,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742282_1458 (size=12001) 2024-12-05T05:18:22,962 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/33fc6b27327542c4b0d2755a1876d2de 2024-12-05T05:18:22,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/8237070e81664815bc4ca441fd3a7b82 is 50, key is test_row_0/C:col10/1733375902367/Put/seqid=0 2024-12-05T05:18:22,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742283_1459 (size=12001) 2024-12-05T05:18:22,978 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/8237070e81664815bc4ca441fd3a7b82 2024-12-05T05:18:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/83033ab833b140d4949e1a297553c852 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/83033ab833b140d4949e1a297553c852 2024-12-05T05:18:22,987 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/83033ab833b140d4949e1a297553c852, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T05:18:22,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/33fc6b27327542c4b0d2755a1876d2de as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/33fc6b27327542c4b0d2755a1876d2de 2024-12-05T05:18:22,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375962981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:22,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375962983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:22,996 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:22,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375962984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375963003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,023 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/33fc6b27327542c4b0d2755a1876d2de, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T05:18:23,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/8237070e81664815bc4ca441fd3a7b82 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8237070e81664815bc4ca441fd3a7b82 2024-12-05T05:18:23,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375963013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,029 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8237070e81664815bc4ca441fd3a7b82, entries=150, sequenceid=47, filesize=11.7 K 2024-12-05T05:18:23,030 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 41b82809bb37e31af4591859446916bc in 163ms, sequenceid=47, compaction requested=false 2024-12-05T05:18:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:23,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-12-05T05:18:23,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-12-05T05:18:23,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-05T05:18:23,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 778 msec 2024-12-05T05:18:23,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 786 msec 2024-12-05T05:18:23,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:23,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T05:18:23,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:23,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:23,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:23,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:23,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:23,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:23,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b59360551b9c42aea00426be3653c400 is 50, key is test_row_0/A:col10/1733375902981/Put/seqid=0 2024-12-05T05:18:23,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742284_1460 (size=14341) 2024-12-05T05:18:23,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b59360551b9c42aea00426be3653c400 2024-12-05T05:18:23,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/84a5912fbca544f28183a97a3a9fbbb7 is 50, key is test_row_0/B:col10/1733375902981/Put/seqid=0 2024-12-05T05:18:23,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375903166, max timestamp=1733375903198Num uniques 859; 2024-12-05T05:18:23,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:23,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375903166, max timestamp=1733375903198Num uniques 859; 2024-12-05T05:18:23,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:23,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375903166, max timestamp=1733375903198Num uniques 859; 2024-12-05T05:18:23,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:23,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742285_1461 (size=12001) 2024-12-05T05:18:23,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375963264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375963267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375963269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375963383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375963383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-05T05:18:23,393 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:23,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-12-05T05:18:23,395 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T05:18:23,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:23,396 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:23,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375963393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T05:18:23,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375963523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375963543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-05T05:18:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:23,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375963593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375963604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/84a5912fbca544f28183a97a3a9fbbb7 2024-12-05T05:18:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/1cdf29717d0f4bedad32d0ab600ea899 is 50, key is test_row_0/C:col10/1733375902981/Put/seqid=0 2024-12-05T05:18:23,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742286_1462 (size=12001) 2024-12-05T05:18:23,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375963624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T05:18:23,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-05T05:18:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:23,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-05T05:18:23,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:23,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:23,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:23,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:23,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375963914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375963914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,926 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:23,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:23,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375963942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:23,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T05:18:24,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-05T05:18:24,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:24,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:24,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:24,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:24,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:24,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:24,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/1cdf29717d0f4bedad32d0ab600ea899 2024-12-05T05:18:24,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b59360551b9c42aea00426be3653c400 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b59360551b9c42aea00426be3653c400 2024-12-05T05:18:24,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b59360551b9c42aea00426be3653c400, entries=200, sequenceid=59, filesize=14.0 K 2024-12-05T05:18:24,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/84a5912fbca544f28183a97a3a9fbbb7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/84a5912fbca544f28183a97a3a9fbbb7 2024-12-05T05:18:24,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/84a5912fbca544f28183a97a3a9fbbb7, entries=150, sequenceid=59, filesize=11.7 K 2024-12-05T05:18:24,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/1cdf29717d0f4bedad32d0ab600ea899 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1cdf29717d0f4bedad32d0ab600ea899 2024-12-05T05:18:24,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1cdf29717d0f4bedad32d0ab600ea899, entries=150, sequenceid=59, filesize=11.7 K 2024-12-05T05:18:24,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 41b82809bb37e31af4591859446916bc in 943ms, sequenceid=59, compaction requested=true 2024-12-05T05:18:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:24,052 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:24,052 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:24,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:24,053 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:24,053 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:24,053 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:24,053 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:24,053 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:24,053 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:24,053 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/7707a59ce9f447a993583fc7241d2f41, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/33fc6b27327542c4b0d2755a1876d2de, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/84a5912fbca544f28183a97a3a9fbbb7] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.2 K 2024-12-05T05:18:24,053 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8a7530a0c1784220901ba434fe1a82a9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/83033ab833b140d4949e1a297553c852, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b59360551b9c42aea00426be3653c400] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=39.7 K 2024-12-05T05:18:24,054 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a7530a0c1784220901ba434fe1a82a9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733375902256 2024-12-05T05:18:24,054 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7707a59ce9f447a993583fc7241d2f41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733375902256 2024-12-05T05:18:24,054 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83033ab833b140d4949e1a297553c852, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733375902356 2024-12-05T05:18:24,054 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 33fc6b27327542c4b0d2755a1876d2de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733375902356 2024-12-05T05:18:24,054 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59360551b9c42aea00426be3653c400, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733375902967 2024-12-05T05:18:24,054 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 84a5912fbca544f28183a97a3a9fbbb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733375902981 2024-12-05T05:18:24,073 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:24,074 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/d60f86e9d80240f382c2c6b4ef07a6de is 50, key is test_row_0/B:col10/1733375902981/Put/seqid=0 2024-12-05T05:18:24,075 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#384 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:24,076 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b3ef69ce710b467081c3a1fe1eb603d2 is 50, key is test_row_0/A:col10/1733375902981/Put/seqid=0 2024-12-05T05:18:24,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742287_1463 (size=12104) 2024-12-05T05:18:24,093 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/d60f86e9d80240f382c2c6b4ef07a6de as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/d60f86e9d80240f382c2c6b4ef07a6de 2024-12-05T05:18:24,099 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into d60f86e9d80240f382c2c6b4ef07a6de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:24,099 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:24,099 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=13, startTime=1733375904052; duration=0sec 2024-12-05T05:18:24,099 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:24,099 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:24,099 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:24,100 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:24,100 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:24,100 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:24,100 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a19662f599f24d8d8e57fd41a5bfcd1c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8237070e81664815bc4ca441fd3a7b82, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1cdf29717d0f4bedad32d0ab600ea899] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.2 K 2024-12-05T05:18:24,101 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a19662f599f24d8d8e57fd41a5bfcd1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733375902256 2024-12-05T05:18:24,101 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8237070e81664815bc4ca441fd3a7b82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733375902356 2024-12-05T05:18:24,101 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cdf29717d0f4bedad32d0ab600ea899, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733375902981 2024-12-05T05:18:24,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742288_1464 (size=12104) 2024-12-05T05:18:24,114 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:24,114 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/15d7acb64e8a4a0d8c6724f7f5f40fa5 is 50, key is test_row_0/C:col10/1733375902981/Put/seqid=0 2024-12-05T05:18:24,122 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b3ef69ce710b467081c3a1fe1eb603d2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b3ef69ce710b467081c3a1fe1eb603d2 2024-12-05T05:18:24,126 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into b3ef69ce710b467081c3a1fe1eb603d2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:24,126 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:24,126 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=13, startTime=1733375904052; duration=0sec 2024-12-05T05:18:24,126 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:24,126 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:24,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742289_1465 (size=12104) 2024-12-05T05:18:24,133 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/15d7acb64e8a4a0d8c6724f7f5f40fa5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/15d7acb64e8a4a0d8c6724f7f5f40fa5 2024-12-05T05:18:24,138 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into 15d7acb64e8a4a0d8c6724f7f5f40fa5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:24,138 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:24,139 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=13, startTime=1733375904052; duration=0sec 2024-12-05T05:18:24,139 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:24,139 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:24,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-12-05T05:18:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:24,177 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=120.76 KB heapSize=340.45 KB 2024-12-05T05:18:24,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:24,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:24,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b4aa0db6ef23410295410e602afac5b3 is 50, key is test_row_0/A:col10/1733375903266/Put/seqid=0 2024-12-05T05:18:24,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742290_1466 (size=12001) 2024-12-05T05:18:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:24,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:24,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375964479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375964479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375964479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T05:18:24,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375964542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,545 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,546 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375964563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,570 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,593 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b4aa0db6ef23410295410e602afac5b3 2024-12-05T05:18:24,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375964592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375964592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/f8512dbae37c4b008ac70b24e44d3e53 is 50, key is test_row_0/B:col10/1733375903266/Put/seqid=0 2024-12-05T05:18:24,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375964603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742291_1467 (size=12001) 2024-12-05T05:18:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375964803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375964803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:24,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375964823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:24,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,015 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/f8512dbae37c4b008ac70b24e44d3e53 2024-12-05T05:18:25,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/ce18a70431c744fea7cd63512294099a is 50, key is test_row_0/C:col10/1733375903266/Put/seqid=0 2024-12-05T05:18:25,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742292_1468 (size=12001) 2024-12-05T05:18:25,113 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T05:18:25,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375965113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375965114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,123 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375965133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,430 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/ce18a70431c744fea7cd63512294099a 2024-12-05T05:18:25,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b4aa0db6ef23410295410e602afac5b3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b4aa0db6ef23410295410e602afac5b3 2024-12-05T05:18:25,437 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b4aa0db6ef23410295410e602afac5b3, entries=150, sequenceid=96, filesize=11.7 K 2024-12-05T05:18:25,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/f8512dbae37c4b008ac70b24e44d3e53 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f8512dbae37c4b008ac70b24e44d3e53 2024-12-05T05:18:25,441 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f8512dbae37c4b008ac70b24e44d3e53, entries=150, sequenceid=96, filesize=11.7 K 2024-12-05T05:18:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/ce18a70431c744fea7cd63512294099a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/ce18a70431c744fea7cd63512294099a 2024-12-05T05:18:25,451 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/ce18a70431c744fea7cd63512294099a, entries=150, sequenceid=96, filesize=11.7 K 2024-12-05T05:18:25,452 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~340.41 KB/348576, currentSize=60.38 KB/61830 for 41b82809bb37e31af4591859446916bc in 1275ms, sequenceid=96, compaction requested=false 2024-12-05T05:18:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-12-05T05:18:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-12-05T05:18:25,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-05T05:18:25,455 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0570 sec 2024-12-05T05:18:25,456 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.0610 sec 2024-12-05T05:18:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-12-05T05:18:25,543 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:25,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-12-05T05:18:25,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T05:18:25,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:25,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:25,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-05T05:18:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:25,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:25,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/a0d544a1184448d988238041f1658b7a is 50, key is test_row_0/A:col10/1733375904477/Put/seqid=0 2024-12-05T05:18:25,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742293_1469 (size=14341) 2024-12-05T05:18:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T05:18:25,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375965670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,680 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375965673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375965680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:25,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:25,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375965782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375965782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375965793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:25,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:25,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T05:18:25,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375965992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375965992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:25,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375966002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/a0d544a1184448d988238041f1658b7a 2024-12-05T05:18:26,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/a874e0a61f2d4a44b77a79a445ba8bed is 50, key is test_row_0/B:col10/1733375904477/Put/seqid=0 2024-12-05T05:18:26,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742294_1470 (size=12001) 2024-12-05T05:18:26,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:26,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T05:18:26,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375966303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375966303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,307 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375966313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/a874e0a61f2d4a44b77a79a445ba8bed 2024-12-05T05:18:26,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/e7cc194d313b492297d94afd397225e2 is 50, key is test_row_0/C:col10/1733375904477/Put/seqid=0 2024-12-05T05:18:26,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742295_1471 (size=12001) 2024-12-05T05:18:26,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375966572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,577 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375966592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,601 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:26,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:26,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T05:18:26,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:26,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:26,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375966812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375966813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:26,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375966822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:26,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:26,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/e7cc194d313b492297d94afd397225e2 2024-12-05T05:18:26,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/a0d544a1184448d988238041f1658b7a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a0d544a1184448d988238041f1658b7a 2024-12-05T05:18:26,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a0d544a1184448d988238041f1658b7a, entries=200, sequenceid=109, filesize=14.0 K 2024-12-05T05:18:26,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/a874e0a61f2d4a44b77a79a445ba8bed as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a874e0a61f2d4a44b77a79a445ba8bed 2024-12-05T05:18:26,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a874e0a61f2d4a44b77a79a445ba8bed, entries=150, sequenceid=109, filesize=11.7 K 2024-12-05T05:18:26,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/e7cc194d313b492297d94afd397225e2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e7cc194d313b492297d94afd397225e2 2024-12-05T05:18:26,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e7cc194d313b492297d94afd397225e2, entries=150, sequenceid=109, filesize=11.7 K 2024-12-05T05:18:26,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 41b82809bb37e31af4591859446916bc in 1254ms, sequenceid=109, compaction requested=true 2024-12-05T05:18:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:26,890 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:26,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:26,890 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:26,891 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:26,891 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:26,891 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:26,891 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:26,891 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,891 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,891 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b3ef69ce710b467081c3a1fe1eb603d2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b4aa0db6ef23410295410e602afac5b3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a0d544a1184448d988238041f1658b7a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=37.5 K 2024-12-05T05:18:26,891 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/d60f86e9d80240f382c2c6b4ef07a6de, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f8512dbae37c4b008ac70b24e44d3e53, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a874e0a61f2d4a44b77a79a445ba8bed] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.3 K 2024-12-05T05:18:26,891 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d60f86e9d80240f382c2c6b4ef07a6de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733375902981 2024-12-05T05:18:26,891 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ef69ce710b467081c3a1fe1eb603d2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733375902981 2024-12-05T05:18:26,891 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4aa0db6ef23410295410e602afac5b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733375903244 2024-12-05T05:18:26,891 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f8512dbae37c4b008ac70b24e44d3e53, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733375903244 2024-12-05T05:18:26,892 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0d544a1184448d988238041f1658b7a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375904452 2024-12-05T05:18:26,892 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a874e0a61f2d4a44b77a79a445ba8bed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375904477 2024-12-05T05:18:26,896 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#393 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:26,897 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/5a56214cf07743f6b4601462e004bc95 is 50, key is test_row_0/A:col10/1733375904477/Put/seqid=0 2024-12-05T05:18:26,898 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:26,899 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/727781dc5312413ca7a9ef5375c51b60 is 50, key is test_row_0/B:col10/1733375904477/Put/seqid=0 2024-12-05T05:18:26,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742296_1472 (size=12207) 2024-12-05T05:18:26,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742297_1473 (size=12207) 2024-12-05T05:18:26,904 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/5a56214cf07743f6b4601462e004bc95 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/5a56214cf07743f6b4601462e004bc95 2024-12-05T05:18:26,907 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into 5a56214cf07743f6b4601462e004bc95(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:26,907 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:26,907 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=13, startTime=1733375906890; duration=0sec 2024-12-05T05:18:26,907 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:26,907 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:26,907 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:26,908 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:26,908 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:26,908 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,908 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/15d7acb64e8a4a0d8c6724f7f5f40fa5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/ce18a70431c744fea7cd63512294099a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e7cc194d313b492297d94afd397225e2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.3 K 2024-12-05T05:18:26,908 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15d7acb64e8a4a0d8c6724f7f5f40fa5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1733375902981 2024-12-05T05:18:26,908 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce18a70431c744fea7cd63512294099a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733375903244 2024-12-05T05:18:26,908 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7cc194d313b492297d94afd397225e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375904477 2024-12-05T05:18:26,914 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:26,914 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/f880c07936eb4143b638aef1ceb74bc1 is 50, key is test_row_0/C:col10/1733375904477/Put/seqid=0 2024-12-05T05:18:26,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742298_1474 (size=12207) 2024-12-05T05:18:26,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-12-05T05:18:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:26,922 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:18:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:26,933 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/f880c07936eb4143b638aef1ceb74bc1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/f880c07936eb4143b638aef1ceb74bc1 2024-12-05T05:18:26,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/cc5e42e05bfb427e8d30715a84786a2c is 50, key is test_row_0/A:col10/1733375905671/Put/seqid=0 2024-12-05T05:18:26,941 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into f880c07936eb4143b638aef1ceb74bc1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:26,942 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:26,942 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=13, startTime=1733375906890; duration=0sec 2024-12-05T05:18:26,942 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:26,942 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:26,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742299_1475 (size=12001) 2024-12-05T05:18:27,307 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/727781dc5312413ca7a9ef5375c51b60 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/727781dc5312413ca7a9ef5375c51b60 2024-12-05T05:18:27,310 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into 727781dc5312413ca7a9ef5375c51b60(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:27,310 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:27,310 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=13, startTime=1733375906890; duration=0sec 2024-12-05T05:18:27,310 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:27,310 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:27,344 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/cc5e42e05bfb427e8d30715a84786a2c 2024-12-05T05:18:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/0d54e577896b4ea59a490edb41882f25 is 50, key is test_row_0/B:col10/1733375905671/Put/seqid=0 2024-12-05T05:18:27,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742300_1476 (size=12001) 2024-12-05T05:18:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T05:18:27,753 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/0d54e577896b4ea59a490edb41882f25 2024-12-05T05:18:27,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/2164ffedc7ec4dad97b8421084737057 is 50, key is test_row_0/C:col10/1733375905671/Put/seqid=0 2024-12-05T05:18:27,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742301_1477 (size=12001) 2024-12-05T05:18:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:27,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:27,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375967851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,861 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375967853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:27,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375967854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:27,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375967963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:27,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375967973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375967973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,165 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/2164ffedc7ec4dad97b8421084737057 2024-12-05T05:18:28,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/cc5e42e05bfb427e8d30715a84786a2c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/cc5e42e05bfb427e8d30715a84786a2c 2024-12-05T05:18:28,171 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/cc5e42e05bfb427e8d30715a84786a2c, entries=150, sequenceid=129, filesize=11.7 K 2024-12-05T05:18:28,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/0d54e577896b4ea59a490edb41882f25 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/0d54e577896b4ea59a490edb41882f25 2024-12-05T05:18:28,174 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/0d54e577896b4ea59a490edb41882f25, entries=150, sequenceid=129, filesize=11.7 K 2024-12-05T05:18:28,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/2164ffedc7ec4dad97b8421084737057 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/2164ffedc7ec4dad97b8421084737057 2024-12-05T05:18:28,177 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/2164ffedc7ec4dad97b8421084737057, entries=150, sequenceid=129, filesize=11.7 K 2024-12-05T05:18:28,178 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 41b82809bb37e31af4591859446916bc in 1256ms, sequenceid=129, compaction requested=false 2024-12-05T05:18:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:28,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-05T05:18:28,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-05T05:18:28,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-05T05:18:28,180 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-12-05T05:18:28,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.6370 sec 2024-12-05T05:18:28,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:28,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T05:18:28,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:28,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:28,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:28,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:28,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:28,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:28,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/6596afd3ad1447778ce1e7eb1986b10b is 50, key is test_row_0/A:col10/1733375907848/Put/seqid=0 2024-12-05T05:18:28,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742302_1478 (size=14541) 2024-12-05T05:18:28,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375968226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375968234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375968234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375968343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375968343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375968343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,351 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375968566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375968566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375968566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/6596afd3ad1447778ce1e7eb1986b10b 2024-12-05T05:18:28,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/f64f3d662c3f4a99935a664c04dd3da9 is 50, key is test_row_0/B:col10/1733375907848/Put/seqid=0 2024-12-05T05:18:28,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742303_1479 (size=12151) 2024-12-05T05:18:28,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375968883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375968883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,884 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375968883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:28,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:29,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/f64f3d662c3f4a99935a664c04dd3da9 2024-12-05T05:18:29,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/a73f21ffab434168a5a498a77cba420e is 50, key is test_row_0/C:col10/1733375907848/Put/seqid=0 2024-12-05T05:18:29,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742304_1480 (size=12151) 2024-12-05T05:18:29,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375969393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:29,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:29,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:29,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:29,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375969393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:29,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375969393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:29,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:29,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/a73f21ffab434168a5a498a77cba420e 2024-12-05T05:18:29,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/6596afd3ad1447778ce1e7eb1986b10b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6596afd3ad1447778ce1e7eb1986b10b 2024-12-05T05:18:29,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6596afd3ad1447778ce1e7eb1986b10b, entries=200, sequenceid=145, filesize=14.2 K 2024-12-05T05:18:29,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/f64f3d662c3f4a99935a664c04dd3da9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f64f3d662c3f4a99935a664c04dd3da9 2024-12-05T05:18:29,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f64f3d662c3f4a99935a664c04dd3da9, entries=150, sequenceid=145, filesize=11.9 K 2024-12-05T05:18:29,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/a73f21ffab434168a5a498a77cba420e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a73f21ffab434168a5a498a77cba420e 2024-12-05T05:18:29,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a73f21ffab434168a5a498a77cba420e, entries=150, sequenceid=145, filesize=11.9 K 2024-12-05T05:18:29,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 41b82809bb37e31af4591859446916bc in 1245ms, sequenceid=145, compaction requested=true 2024-12-05T05:18:29,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:29,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:29,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:29,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:29,435 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:29,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:29,435 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:29,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:29,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38749 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:29,436 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:29,436 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:29,436 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/5a56214cf07743f6b4601462e004bc95, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/cc5e42e05bfb427e8d30715a84786a2c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6596afd3ad1447778ce1e7eb1986b10b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=37.8 K 2024-12-05T05:18:29,436 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/727781dc5312413ca7a9ef5375c51b60, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/0d54e577896b4ea59a490edb41882f25, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f64f3d662c3f4a99935a664c04dd3da9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.5 K 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 727781dc5312413ca7a9ef5375c51b60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375904477 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a56214cf07743f6b4601462e004bc95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375904477 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d54e577896b4ea59a490edb41882f25, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375905655 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5e42e05bfb427e8d30715a84786a2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375905655 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f64f3d662c3f4a99935a664c04dd3da9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733375907848 2024-12-05T05:18:29,436 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6596afd3ad1447778ce1e7eb1986b10b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733375907848 2024-12-05T05:18:29,441 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:29,441 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/9a8edf36460748b191ac145b0aaf27a4 is 50, key is test_row_0/A:col10/1733375907848/Put/seqid=0 2024-12-05T05:18:29,444 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#403 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:29,444 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/cd9a2c6d3fd54e4594564d1ea2d5546b is 50, key is test_row_0/B:col10/1733375907848/Put/seqid=0 2024-12-05T05:18:29,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742306_1482 (size=12459) 2024-12-05T05:18:29,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742305_1481 (size=12459) 2024-12-05T05:18:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-05T05:18:29,713 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:29,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-12-05T05:18:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T05:18:29,715 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:29,715 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:29,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T05:18:29,858 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/cd9a2c6d3fd54e4594564d1ea2d5546b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/cd9a2c6d3fd54e4594564d1ea2d5546b 2024-12-05T05:18:29,858 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/9a8edf36460748b191ac145b0aaf27a4 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/9a8edf36460748b191ac145b0aaf27a4 2024-12-05T05:18:29,862 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into 9a8edf36460748b191ac145b0aaf27a4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:29,862 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into cd9a2c6d3fd54e4594564d1ea2d5546b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:29,862 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:29,862 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:29,862 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=13, startTime=1733375909435; duration=0sec 2024-12-05T05:18:29,862 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=13, startTime=1733375909435; duration=0sec 2024-12-05T05:18:29,862 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:29,862 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:29,862 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:29,863 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:29,863 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:29,863 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:29,863 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:29,863 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:29,863 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/f880c07936eb4143b638aef1ceb74bc1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/2164ffedc7ec4dad97b8421084737057, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a73f21ffab434168a5a498a77cba420e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.5 K 2024-12-05T05:18:29,864 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f880c07936eb4143b638aef1ceb74bc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1733375904477 2024-12-05T05:18:29,864 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2164ffedc7ec4dad97b8421084737057, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1733375905655 2024-12-05T05:18:29,864 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a73f21ffab434168a5a498a77cba420e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733375907848 2024-12-05T05:18:29,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-12-05T05:18:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:29,867 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:18:29,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:29,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:29,870 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:29,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/675d36d061c94e86898952babc6a0be7 is 50, key is test_row_0/A:col10/1733375908232/Put/seqid=0 2024-12-05T05:18:29,871 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/480403018b64477491494609b4efb974 is 50, key is test_row_0/C:col10/1733375907848/Put/seqid=0 2024-12-05T05:18:29,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742307_1483 (size=12459) 2024-12-05T05:18:29,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742308_1484 (size=12151) 2024-12-05T05:18:29,882 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/675d36d061c94e86898952babc6a0be7 2024-12-05T05:18:29,885 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/480403018b64477491494609b4efb974 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/480403018b64477491494609b4efb974 2024-12-05T05:18:29,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/53888fce32304564a5060ff27de707c7 is 50, key is test_row_0/B:col10/1733375908232/Put/seqid=0 2024-12-05T05:18:29,892 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into 480403018b64477491494609b4efb974(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:29,892 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:29,892 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=13, startTime=1733375909435; duration=0sec 2024-12-05T05:18:29,893 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:29,893 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:29,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742309_1485 (size=12151) 2024-12-05T05:18:30,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T05:18:30,296 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/53888fce32304564a5060ff27de707c7 2024-12-05T05:18:30,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/312a457f8b8241a6be932fe195ce4221 is 50, key is test_row_0/C:col10/1733375908232/Put/seqid=0 2024-12-05T05:18:30,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742310_1486 (size=12151) 2024-12-05T05:18:30,321 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/312a457f8b8241a6be932fe195ce4221 2024-12-05T05:18:30,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/675d36d061c94e86898952babc6a0be7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/675d36d061c94e86898952babc6a0be7 2024-12-05T05:18:30,327 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/675d36d061c94e86898952babc6a0be7, entries=150, sequenceid=166, filesize=11.9 K 2024-12-05T05:18:30,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/53888fce32304564a5060ff27de707c7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/53888fce32304564a5060ff27de707c7 2024-12-05T05:18:30,334 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/53888fce32304564a5060ff27de707c7, entries=150, sequenceid=166, filesize=11.9 K 2024-12-05T05:18:30,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/312a457f8b8241a6be932fe195ce4221 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/312a457f8b8241a6be932fe195ce4221 2024-12-05T05:18:30,341 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/312a457f8b8241a6be932fe195ce4221, entries=150, sequenceid=166, filesize=11.9 K 2024-12-05T05:18:30,342 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 41b82809bb37e31af4591859446916bc in 475ms, sequenceid=166, compaction requested=false 2024-12-05T05:18:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:30,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-05T05:18:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-05T05:18:30,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T05:18:30,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-05T05:18:30,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 629 msec 2024-12-05T05:18:30,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 636 msec 2024-12-05T05:18:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:30,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:18:30,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:30,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:30,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:30,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:30,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:30,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:30,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/a5f4f04d4dde4bea9dd14606da5b39da is 50, key is test_row_0/A:col10/1733375910415/Put/seqid=0 2024-12-05T05:18:30,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742311_1487 (size=12151) 2024-12-05T05:18:30,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375970468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375910461, max timestamp=1733375910476Num uniques 859; 2024-12-05T05:18:30,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375910461, max timestamp=1733375910476Num uniques 859; 2024-12-05T05:18:30,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:18:30,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:18:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,480 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375910461, max timestamp=1733375910476Num uniques 859; 2024-12-05T05:18:30,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:18:30,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375970566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375970576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375970583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375970592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,603 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,604 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375970623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,628 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375970683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375970683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375970793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/a5f4f04d4dde4bea9dd14606da5b39da 2024-12-05T05:18:30,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/c6f18c47ca724619b81add2c0ae020ad is 50, key is test_row_0/B:col10/1733375910415/Put/seqid=0 2024-12-05T05:18:30,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742312_1488 (size=12151) 2024-12-05T05:18:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-05T05:18:30,853 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:30,853 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:30,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-05T05:18:30,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T05:18:30,855 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:30,855 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:30,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:30,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375970893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375970893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:30,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:18:30,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T05:18:30,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T05:18:31,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T05:18:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:31,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:31,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:31,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375971113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:31,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T05:18:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:31,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T05:18:31,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375971203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:31,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375971203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:31,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/c6f18c47ca724619b81add2c0ae020ad 2024-12-05T05:18:31,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/719e842cf3ff4ce791bfe8d6bc9b0749 is 50, key is test_row_0/C:col10/1733375910415/Put/seqid=0 2024-12-05T05:18:31,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742313_1489 (size=12151) 2024-12-05T05:18:31,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=178 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/719e842cf3ff4ce791bfe8d6bc9b0749 2024-12-05T05:18:31,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/a5f4f04d4dde4bea9dd14606da5b39da as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a5f4f04d4dde4bea9dd14606da5b39da 2024-12-05T05:18:31,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a5f4f04d4dde4bea9dd14606da5b39da, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T05:18:31,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/c6f18c47ca724619b81add2c0ae020ad as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c6f18c47ca724619b81add2c0ae020ad 2024-12-05T05:18:31,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c6f18c47ca724619b81add2c0ae020ad, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T05:18:31,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/719e842cf3ff4ce791bfe8d6bc9b0749 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/719e842cf3ff4ce791bfe8d6bc9b0749 2024-12-05T05:18:31,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/719e842cf3ff4ce791bfe8d6bc9b0749, entries=150, sequenceid=178, filesize=11.9 K 2024-12-05T05:18:31,273 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 41b82809bb37e31af4591859446916bc in 857ms, sequenceid=178, compaction requested=true 2024-12-05T05:18:31,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:31,273 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:31,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:31,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:31,273 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:31,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:31,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:31,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:31,274 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:31,274 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:31,274 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:31,274 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:31,275 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/9a8edf36460748b191ac145b0aaf27a4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/675d36d061c94e86898952babc6a0be7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a5f4f04d4dde4bea9dd14606da5b39da] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.9 K 2024-12-05T05:18:31,275 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:31,275 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:31,275 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:31,275 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/cd9a2c6d3fd54e4594564d1ea2d5546b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/53888fce32304564a5060ff27de707c7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c6f18c47ca724619b81add2c0ae020ad] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.9 K 2024-12-05T05:18:31,275 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a8edf36460748b191ac145b0aaf27a4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733375907848 2024-12-05T05:18:31,276 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting cd9a2c6d3fd54e4594564d1ea2d5546b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733375907848 2024-12-05T05:18:31,277 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 675d36d061c94e86898952babc6a0be7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733375908219 2024-12-05T05:18:31,277 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 53888fce32304564a5060ff27de707c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733375908219 2024-12-05T05:18:31,278 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f4f04d4dde4bea9dd14606da5b39da, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733375910408 2024-12-05T05:18:31,278 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f18c47ca724619b81add2c0ae020ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733375910408 2024-12-05T05:18:31,285 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:31,286 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/87945ab8ae094ad3a5b95ac483d35cf4 is 50, key is test_row_0/B:col10/1733375910415/Put/seqid=0 2024-12-05T05:18:31,290 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#412 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:31,290 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/a05c9b119e5e476d96b301c6d35f8dc0 is 50, key is test_row_0/A:col10/1733375910415/Put/seqid=0 2024-12-05T05:18:31,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742314_1490 (size=12561) 2024-12-05T05:18:31,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-05T05:18:31,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:31,314 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-05T05:18:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:31,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:31,319 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/87945ab8ae094ad3a5b95ac483d35cf4 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/87945ab8ae094ad3a5b95ac483d35cf4 2024-12-05T05:18:31,326 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into 87945ab8ae094ad3a5b95ac483d35cf4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:31,326 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:31,326 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=13, startTime=1733375911273; duration=0sec 2024-12-05T05:18:31,327 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:31,327 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:31,327 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:31,327 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:31,327 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:31,327 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:31,328 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/480403018b64477491494609b4efb974, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/312a457f8b8241a6be932fe195ce4221, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/719e842cf3ff4ce791bfe8d6bc9b0749] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=35.9 K 2024-12-05T05:18:31,328 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 480403018b64477491494609b4efb974, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1733375907848 2024-12-05T05:18:31,328 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 312a457f8b8241a6be932fe195ce4221, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733375908219 2024-12-05T05:18:31,328 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 719e842cf3ff4ce791bfe8d6bc9b0749, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733375910408 2024-12-05T05:18:31,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742315_1491 (size=12561) 2024-12-05T05:18:31,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/1ccddf15026c40b1bc096d70b05e0186 is 50, key is test_row_0/A:col10/1733375910574/Put/seqid=0 2024-12-05T05:18:31,342 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/a05c9b119e5e476d96b301c6d35f8dc0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a05c9b119e5e476d96b301c6d35f8dc0 2024-12-05T05:18:31,346 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into a05c9b119e5e476d96b301c6d35f8dc0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:31,346 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:31,346 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=13, startTime=1733375911273; duration=0sec 2024-12-05T05:18:31,346 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:31,346 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:31,355 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:31,356 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/dca9652232074a399050b244b3115428 is 50, key is test_row_0/C:col10/1733375910415/Put/seqid=0 2024-12-05T05:18:31,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742316_1492 (size=12151) 2024-12-05T05:18:31,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742317_1493 (size=12561) 2024-12-05T05:18:31,404 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/dca9652232074a399050b244b3115428 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/dca9652232074a399050b244b3115428 2024-12-05T05:18:31,409 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into dca9652232074a399050b244b3115428(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:31,409 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:31,409 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=13, startTime=1733375911274; duration=0sec 2024-12-05T05:18:31,409 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:31,409 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:31,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T05:18:31,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:31,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:31,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:31,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375971680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375971723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:31,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375971723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,728 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:31,758 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/1ccddf15026c40b1bc096d70b05e0186 2024-12-05T05:18:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/b718bdb407ca4a3790ba6679c86fc66a is 50, key is test_row_0/B:col10/1733375910574/Put/seqid=0 2024-12-05T05:18:31,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742318_1494 (size=12151) 2024-12-05T05:18:31,783 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/b718bdb407ca4a3790ba6679c86fc66a 2024-12-05T05:18:31,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/438b8891bcb54487875261e6632c5b05 is 50, key is test_row_0/C:col10/1733375910574/Put/seqid=0 2024-12-05T05:18:31,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:31,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375971793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:31,802 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:31,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742319_1495 (size=12151) 2024-12-05T05:18:31,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T05:18:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375972013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:32,210 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/438b8891bcb54487875261e6632c5b05 2024-12-05T05:18:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/1ccddf15026c40b1bc096d70b05e0186 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/1ccddf15026c40b1bc096d70b05e0186 2024-12-05T05:18:32,217 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/1ccddf15026c40b1bc096d70b05e0186, entries=150, sequenceid=214, filesize=11.9 K 2024-12-05T05:18:32,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/b718bdb407ca4a3790ba6679c86fc66a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/b718bdb407ca4a3790ba6679c86fc66a 2024-12-05T05:18:32,221 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/b718bdb407ca4a3790ba6679c86fc66a, entries=150, sequenceid=214, filesize=11.9 K 2024-12-05T05:18:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-12-05T05:18:32,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/438b8891bcb54487875261e6632c5b05 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/438b8891bcb54487875261e6632c5b05 2024-12-05T05:18:32,225 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/438b8891bcb54487875261e6632c5b05, entries=150, sequenceid=214, filesize=11.9 K 2024-12-05T05:18:32,225 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 41b82809bb37e31af4591859446916bc in 911ms, sequenceid=214, compaction requested=false 2024-12-05T05:18:32,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:32,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:32,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-05T05:18:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-05T05:18:32,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-05T05:18:32,228 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3700 sec 2024-12-05T05:18:32,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.3750 sec 2024-12-05T05:18:32,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:32,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T05:18:32,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:32,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:32,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:32,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:32,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:32,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:32,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/c3b9619fdbd942db807a948ee82a53f5 is 50, key is test_row_0/A:col10/1733375911637/Put/seqid=0 2024-12-05T05:18:32,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742320_1496 (size=14541) 2024-12-05T05:18:32,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:32,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375972450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:32,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375972562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:32,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/c3b9619fdbd942db807a948ee82a53f5 2024-12-05T05:18:32,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375972733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:32,740 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:32,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:32,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/9b5d64bf3b104201a4a4d558e7ad3a27 is 50, key is test_row_0/B:col10/1733375911637/Put/seqid=0 2024-12-05T05:18:32,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:32,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375972742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:32,751 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:32,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742321_1497 (size=12151) 2024-12-05T05:18:32,773 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/9b5d64bf3b104201a4a4d558e7ad3a27 2024-12-05T05:18:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375972773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:32,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:32,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/fa61a0bce3684d60af41b3efec3e1bed is 50, key is test_row_0/C:col10/1733375911637/Put/seqid=0 2024-12-05T05:18:32,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742322_1498 (size=12151) 2024-12-05T05:18:32,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/fa61a0bce3684d60af41b3efec3e1bed 2024-12-05T05:18:32,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/c3b9619fdbd942db807a948ee82a53f5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/c3b9619fdbd942db807a948ee82a53f5 2024-12-05T05:18:32,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/c3b9619fdbd942db807a948ee82a53f5, entries=200, sequenceid=229, filesize=14.2 K 2024-12-05T05:18:32,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/9b5d64bf3b104201a4a4d558e7ad3a27 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9b5d64bf3b104201a4a4d558e7ad3a27 2024-12-05T05:18:32,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9b5d64bf3b104201a4a4d558e7ad3a27, entries=150, sequenceid=229, filesize=11.9 K 2024-12-05T05:18:32,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/fa61a0bce3684d60af41b3efec3e1bed as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/fa61a0bce3684d60af41b3efec3e1bed 2024-12-05T05:18:32,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/fa61a0bce3684d60af41b3efec3e1bed, entries=150, sequenceid=229, filesize=11.9 K 2024-12-05T05:18:32,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 41b82809bb37e31af4591859446916bc in 511ms, sequenceid=229, compaction requested=true 2024-12-05T05:18:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:32,837 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:32,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:32,837 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:32,838 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:32,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:32,838 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:32,838 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:32,838 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a05c9b119e5e476d96b301c6d35f8dc0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/1ccddf15026c40b1bc096d70b05e0186, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/c3b9619fdbd942db807a948ee82a53f5] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=38.3 K 2024-12-05T05:18:32,838 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:32,838 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:32,838 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:32,838 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/87945ab8ae094ad3a5b95ac483d35cf4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/b718bdb407ca4a3790ba6679c86fc66a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9b5d64bf3b104201a4a4d558e7ad3a27] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=36.0 K 2024-12-05T05:18:32,839 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting a05c9b119e5e476d96b301c6d35f8dc0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733375910408 2024-12-05T05:18:32,839 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 87945ab8ae094ad3a5b95ac483d35cf4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733375910408 2024-12-05T05:18:32,839 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ccddf15026c40b1bc096d70b05e0186, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733375910553 2024-12-05T05:18:32,839 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b718bdb407ca4a3790ba6679c86fc66a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733375910553 2024-12-05T05:18:32,839 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3b9619fdbd942db807a948ee82a53f5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375911637 2024-12-05T05:18:32,839 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b5d64bf3b104201a4a4d558e7ad3a27, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375911637 2024-12-05T05:18:32,862 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:32,862 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/2ff3579063a5481982df16e896af6d58 is 50, key is test_row_0/A:col10/1733375911637/Put/seqid=0 2024-12-05T05:18:32,869 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:32,870 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/549a11cc29414c8dbb1ed67575735877 is 50, key is test_row_0/B:col10/1733375911637/Put/seqid=0 2024-12-05T05:18:32,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742323_1499 (size=12663) 2024-12-05T05:18:32,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742324_1500 (size=12663) 2024-12-05T05:18:33,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-05T05:18:33,003 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:33,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-05T05:18:33,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T05:18:33,007 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:33,007 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:33,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:33,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T05:18:33,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:33,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:33,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:33,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:33,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:33,091 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:33,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/ff6bcef8685942968441087866072833 is 50, key is test_row_0/A:col10/1733375913089/Put/seqid=0 2024-12-05T05:18:33,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742325_1501 (size=14541) 2024-12-05T05:18:33,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/ff6bcef8685942968441087866072833 2024-12-05T05:18:33,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T05:18:33,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/8a912f20caa9464c806eafea5ad2ed20 is 50, key is test_row_0/B:col10/1733375913089/Put/seqid=0 2024-12-05T05:18:33,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742326_1502 (size=12151) 2024-12-05T05:18:33,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T05:18:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:33,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375973158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:33,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:33,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:33,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375973272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:33,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:33,301 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/2ff3579063a5481982df16e896af6d58 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/2ff3579063a5481982df16e896af6d58 2024-12-05T05:18:33,306 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/549a11cc29414c8dbb1ed67575735877 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/549a11cc29414c8dbb1ed67575735877 2024-12-05T05:18:33,307 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into 2ff3579063a5481982df16e896af6d58(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:33,307 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:33,307 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=13, startTime=1733375912837; duration=0sec 2024-12-05T05:18:33,308 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:33,308 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:33,308 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:33,311 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:33,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T05:18:33,316 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:33,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:33,316 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:33,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:33,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:33,316 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/dca9652232074a399050b244b3115428, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/438b8891bcb54487875261e6632c5b05, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/fa61a0bce3684d60af41b3efec3e1bed] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=36.0 K 2024-12-05T05:18:33,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,317 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca9652232074a399050b244b3115428, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=178, earliestPutTs=1733375910408 2024-12-05T05:18:33,317 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into 549a11cc29414c8dbb1ed67575735877(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:33,317 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:33,317 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=13, startTime=1733375912837; duration=0sec 2024-12-05T05:18:33,317 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:33,317 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:33,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,320 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 438b8891bcb54487875261e6632c5b05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1733375910553 2024-12-05T05:18:33,320 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa61a0bce3684d60af41b3efec3e1bed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375911637 2024-12-05T05:18:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T05:18:33,330 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#424 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:33,330 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/6b0a972b71724a239dc96fb95153c6e8 is 50, key is test_row_0/C:col10/1733375911637/Put/seqid=0 2024-12-05T05:18:33,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742327_1503 (size=12663) 2024-12-05T05:18:33,356 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/6b0a972b71724a239dc96fb95153c6e8 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6b0a972b71724a239dc96fb95153c6e8 2024-12-05T05:18:33,368 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into 6b0a972b71724a239dc96fb95153c6e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:33,368 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:33,368 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=13, startTime=1733375912838; duration=0sec 2024-12-05T05:18:33,368 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:33,368 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:33,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T05:18:33,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:33,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:33,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:33,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:33,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:33,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375973482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:33,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/8a912f20caa9464c806eafea5ad2ed20 2024-12-05T05:18:33,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/99382a0a7ad04744b5c9832c7aa54d7b is 50, key is test_row_0/C:col10/1733375913089/Put/seqid=0 2024-12-05T05:18:33,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742328_1504 (size=12151) 2024-12-05T05:18:33,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/99382a0a7ad04744b5c9832c7aa54d7b 2024-12-05T05:18:33,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/ff6bcef8685942968441087866072833 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/ff6bcef8685942968441087866072833 2024-12-05T05:18:33,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/ff6bcef8685942968441087866072833, entries=200, sequenceid=250, filesize=14.2 K 2024-12-05T05:18:33,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/8a912f20caa9464c806eafea5ad2ed20 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/8a912f20caa9464c806eafea5ad2ed20 2024-12-05T05:18:33,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/8a912f20caa9464c806eafea5ad2ed20, entries=150, sequenceid=250, filesize=11.9 K 2024-12-05T05:18:33,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/99382a0a7ad04744b5c9832c7aa54d7b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/99382a0a7ad04744b5c9832c7aa54d7b 2024-12-05T05:18:33,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/99382a0a7ad04744b5c9832c7aa54d7b, entries=150, sequenceid=250, filesize=11.9 K 2024-12-05T05:18:33,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 41b82809bb37e31af4591859446916bc in 463ms, sequenceid=250, compaction requested=false 2024-12-05T05:18:33,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:33,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-05T05:18:33,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:33,623 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:18:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:33,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/3eea3a8547364bfa8c22cf288615e303 is 50, key is test_row_1/A:col10/1733375913132/Put/seqid=0 2024-12-05T05:18:33,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T05:18:33,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742329_1505 (size=9857) 2024-12-05T05:18:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:33,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:33,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375913868, max timestamp=1733375913928Num uniques 859; 2024-12-05T05:18:33,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:18:33,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:33,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375973963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:33,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:34,036 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/3eea3a8547364bfa8c22cf288615e303 2024-12-05T05:18:34,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/5ea1c5bd0faa4a659fc18e19c574423f is 50, key is test_row_1/B:col10/1733375913132/Put/seqid=0 2024-12-05T05:18:34,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742330_1506 (size=9857) 2024-12-05T05:18:34,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375974072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:34,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T05:18:34,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:34,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375974292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:34,446 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/5ea1c5bd0faa4a659fc18e19c574423f 2024-12-05T05:18:34,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/6a7585bdfbd34a71a35c4bd1874910e7 is 50, key is test_row_1/C:col10/1733375913132/Put/seqid=0 2024-12-05T05:18:34,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742331_1507 (size=9857) 2024-12-05T05:18:34,483 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/6a7585bdfbd34a71a35c4bd1874910e7 2024-12-05T05:18:34,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/3eea3a8547364bfa8c22cf288615e303 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/3eea3a8547364bfa8c22cf288615e303 2024-12-05T05:18:34,488 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/3eea3a8547364bfa8c22cf288615e303, entries=100, sequenceid=264, filesize=9.6 K 2024-12-05T05:18:34,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/5ea1c5bd0faa4a659fc18e19c574423f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/5ea1c5bd0faa4a659fc18e19c574423f 2024-12-05T05:18:34,498 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/5ea1c5bd0faa4a659fc18e19c574423f, entries=100, sequenceid=264, filesize=9.6 K 2024-12-05T05:18:34,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/6a7585bdfbd34a71a35c4bd1874910e7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6a7585bdfbd34a71a35c4bd1874910e7 2024-12-05T05:18:34,535 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6a7585bdfbd34a71a35c4bd1874910e7, entries=100, sequenceid=264, filesize=9.6 K 2024-12-05T05:18:34,536 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 41b82809bb37e31af4591859446916bc in 913ms, sequenceid=264, compaction requested=true 2024-12-05T05:18:34,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:34,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:34,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-05T05:18:34,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-05T05:18:34,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-05T05:18:34,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5300 sec 2024-12-05T05:18:34,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5350 sec 2024-12-05T05:18:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:34,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=129.71 KB heapSize=381.64 KB 2024-12-05T05:18:34,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:34,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:34,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:34,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:34,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:34,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:34,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/4c37a508e82f4abe81a71748361d6681 is 50, key is test_row_0/A:col10/1733375913962/Put/seqid=0 2024-12-05T05:18:34,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742332_1508 (size=14741) 2024-12-05T05:18:34,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375974667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:34,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:34,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375974753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:34,762 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,763 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:34,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375974772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:34,780 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:34,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375974782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,790 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:34,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375974993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:34,998 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:35,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/4c37a508e82f4abe81a71748361d6681 2024-12-05T05:18:35,031 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/9fbd1c3ce32542bcbfc2b9749de8b157 is 50, key is test_row_0/B:col10/1733375913962/Put/seqid=0 2024-12-05T05:18:35,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742333_1509 (size=12301) 2024-12-05T05:18:35,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-05T05:18:35,153 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:35,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:35,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-05T05:18:35,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T05:18:35,155 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:35,156 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:35,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T05:18:35,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T05:18:35,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375975303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:35,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:35,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/9fbd1c3ce32542bcbfc2b9749de8b157 2024-12-05T05:18:35,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/00014b3e3a904a37b6e9a43fa7a10ddd is 50, key is test_row_0/C:col10/1733375913962/Put/seqid=0 2024-12-05T05:18:35,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742334_1510 (size=12301) 2024-12-05T05:18:35,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T05:18:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T05:18:35,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T05:18:35,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:35,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T05:18:35,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:35,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:35,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T05:18:35,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375975823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:35,829 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:35,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/00014b3e3a904a37b6e9a43fa7a10ddd 2024-12-05T05:18:35,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/4c37a508e82f4abe81a71748361d6681 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/4c37a508e82f4abe81a71748361d6681 2024-12-05T05:18:35,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/4c37a508e82f4abe81a71748361d6681, entries=200, sequenceid=291, filesize=14.4 K 2024-12-05T05:18:35,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/9fbd1c3ce32542bcbfc2b9749de8b157 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9fbd1c3ce32542bcbfc2b9749de8b157 2024-12-05T05:18:35,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9fbd1c3ce32542bcbfc2b9749de8b157, entries=150, sequenceid=291, filesize=12.0 K 2024-12-05T05:18:35,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/00014b3e3a904a37b6e9a43fa7a10ddd as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/00014b3e3a904a37b6e9a43fa7a10ddd 2024-12-05T05:18:35,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/00014b3e3a904a37b6e9a43fa7a10ddd, entries=150, sequenceid=291, filesize=12.0 K 2024-12-05T05:18:35,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~381.59 KB/390752, currentSize=46.96 KB/48090 for 41b82809bb37e31af4591859446916bc in 1262ms, sequenceid=291, compaction requested=true 2024-12-05T05:18:35,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:35,872 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:35,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:35,872 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:35,872 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51802 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:35,872 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:35,872 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:35,872 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:35,873 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,873 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,873 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/549a11cc29414c8dbb1ed67575735877, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/8a912f20caa9464c806eafea5ad2ed20, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/5ea1c5bd0faa4a659fc18e19c574423f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9fbd1c3ce32542bcbfc2b9749de8b157] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=45.9 K 2024-12-05T05:18:35,873 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/2ff3579063a5481982df16e896af6d58, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/ff6bcef8685942968441087866072833, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/3eea3a8547364bfa8c22cf288615e303, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/4c37a508e82f4abe81a71748361d6681] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=50.6 K 2024-12-05T05:18:35,873 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 549a11cc29414c8dbb1ed67575735877, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375911637 2024-12-05T05:18:35,873 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ff3579063a5481982df16e896af6d58, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375911637 2024-12-05T05:18:35,873 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a912f20caa9464c806eafea5ad2ed20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733375912385 2024-12-05T05:18:35,873 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff6bcef8685942968441087866072833, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733375912385 2024-12-05T05:18:35,873 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ea1c5bd0faa4a659fc18e19c574423f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733375913132 2024-12-05T05:18:35,873 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eea3a8547364bfa8c22cf288615e303, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733375913132 2024-12-05T05:18:35,873 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fbd1c3ce32542bcbfc2b9749de8b157, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733375913928 2024-12-05T05:18:35,873 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c37a508e82f4abe81a71748361d6681, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733375913928 2024-12-05T05:18:35,880 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#432 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:35,880 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#433 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:35,881 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/7a485868dd32473c88f7d0973e020355 is 50, key is test_row_0/A:col10/1733375913962/Put/seqid=0 2024-12-05T05:18:35,881 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/c54b0d3722b048728249bc06b8be8c0d is 50, key is test_row_0/B:col10/1733375913962/Put/seqid=0 2024-12-05T05:18:35,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742335_1511 (size=12949) 2024-12-05T05:18:35,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742336_1512 (size=12949) 2024-12-05T05:18:35,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-05T05:18:35,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:35,922 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:18:35,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:35,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:35,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:35,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:35,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:35,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/d74237e97ef844668d9d91642a2ea0c1 is 50, key is test_row_0/A:col10/1733375914666/Put/seqid=0 2024-12-05T05:18:35,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742337_1513 (size=9857) 2024-12-05T05:18:36,288 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/c54b0d3722b048728249bc06b8be8c0d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c54b0d3722b048728249bc06b8be8c0d 2024-12-05T05:18:36,292 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into c54b0d3722b048728249bc06b8be8c0d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:36,292 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:36,292 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=12, startTime=1733375915872; duration=0sec 2024-12-05T05:18:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T05:18:36,292 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:36,292 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:36,293 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:36,294 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:36,294 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:36,294 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:36,294 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6b0a972b71724a239dc96fb95153c6e8, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/99382a0a7ad04744b5c9832c7aa54d7b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6a7585bdfbd34a71a35c4bd1874910e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/00014b3e3a904a37b6e9a43fa7a10ddd] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=45.9 K 2024-12-05T05:18:36,294 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0a972b71724a239dc96fb95153c6e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1733375911637 2024-12-05T05:18:36,294 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 99382a0a7ad04744b5c9832c7aa54d7b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1733375912385 2024-12-05T05:18:36,295 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a7585bdfbd34a71a35c4bd1874910e7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1733375913132 2024-12-05T05:18:36,295 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 00014b3e3a904a37b6e9a43fa7a10ddd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733375913928 2024-12-05T05:18:36,304 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/7a485868dd32473c88f7d0973e020355 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/7a485868dd32473c88f7d0973e020355 2024-12-05T05:18:36,305 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#435 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:36,305 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/329e6bb3304146a0a252514c34d48850 is 50, key is test_row_0/C:col10/1733375913962/Put/seqid=0 2024-12-05T05:18:36,309 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into 7a485868dd32473c88f7d0973e020355(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:36,309 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:36,309 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=12, startTime=1733375915872; duration=0sec 2024-12-05T05:18:36,309 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:36,309 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:36,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742338_1514 (size=12949) 2024-12-05T05:18:36,318 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/329e6bb3304146a0a252514c34d48850 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/329e6bb3304146a0a252514c34d48850 2024-12-05T05:18:36,322 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into 329e6bb3304146a0a252514c34d48850(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:36,322 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:36,322 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=12, startTime=1733375915872; duration=0sec 2024-12-05T05:18:36,323 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:36,323 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:36,333 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/d74237e97ef844668d9d91642a2ea0c1 2024-12-05T05:18:36,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/6da0cc56298f4ed8a9b58881cd7ad8b1 is 50, key is test_row_0/B:col10/1733375914666/Put/seqid=0 2024-12-05T05:18:36,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742339_1515 (size=9857) 2024-12-05T05:18:36,752 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/6da0cc56298f4ed8a9b58881cd7ad8b1 2024-12-05T05:18:36,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/1b8c2803f1324f90ac12edaad186f82e is 50, key is test_row_0/C:col10/1733375914666/Put/seqid=0 2024-12-05T05:18:36,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742340_1516 (size=9857) 2024-12-05T05:18:36,774 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/1b8c2803f1324f90ac12edaad186f82e 2024-12-05T05:18:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/d74237e97ef844668d9d91642a2ea0c1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d74237e97ef844668d9d91642a2ea0c1 2024-12-05T05:18:36,784 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d74237e97ef844668d9d91642a2ea0c1, entries=100, sequenceid=301, filesize=9.6 K 2024-12-05T05:18:36,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/6da0cc56298f4ed8a9b58881cd7ad8b1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6da0cc56298f4ed8a9b58881cd7ad8b1 2024-12-05T05:18:36,788 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6da0cc56298f4ed8a9b58881cd7ad8b1, entries=100, sequenceid=301, filesize=9.6 K 2024-12-05T05:18:36,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/1b8c2803f1324f90ac12edaad186f82e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1b8c2803f1324f90ac12edaad186f82e 2024-12-05T05:18:36,793 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1b8c2803f1324f90ac12edaad186f82e, entries=100, sequenceid=301, filesize=9.6 K 2024-12-05T05:18:36,794 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 41b82809bb37e31af4591859446916bc in 872ms, sequenceid=301, compaction requested=false 2024-12-05T05:18:36,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:36,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:36,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-05T05:18:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-05T05:18:36,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-05T05:18:36,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-12-05T05:18:36,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.6450 sec 2024-12-05T05:18:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:36,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:18:36,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:36,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:36,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:36,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:36,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:36,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:36,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/6670dd3b4a5d4e3388deb51808e0450c is 50, key is test_row_0/A:col10/1733375916870/Put/seqid=0 2024-12-05T05:18:36,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742341_1517 (size=12297) 2024-12-05T05:18:36,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/6670dd3b4a5d4e3388deb51808e0450c 2024-12-05T05:18:36,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/09809278003d4690898ae138544d4c4b is 50, key is test_row_0/B:col10/1733375916870/Put/seqid=0 2024-12-05T05:18:36,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742342_1518 (size=9857) 2024-12-05T05:18:36,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/09809278003d4690898ae138544d4c4b 2024-12-05T05:18:36,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/84d09803430347d8a9212e8f657a1962 is 50, key is test_row_0/C:col10/1733375916870/Put/seqid=0 2024-12-05T05:18:36,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742343_1519 (size=9857) 2024-12-05T05:18:36,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/84d09803430347d8a9212e8f657a1962 2024-12-05T05:18:36,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/6670dd3b4a5d4e3388deb51808e0450c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6670dd3b4a5d4e3388deb51808e0450c 2024-12-05T05:18:36,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6670dd3b4a5d4e3388deb51808e0450c, entries=150, sequenceid=314, filesize=12.0 K 2024-12-05T05:18:36,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/09809278003d4690898ae138544d4c4b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/09809278003d4690898ae138544d4c4b 2024-12-05T05:18:36,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/09809278003d4690898ae138544d4c4b, entries=100, sequenceid=314, filesize=9.6 K 2024-12-05T05:18:36,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/84d09803430347d8a9212e8f657a1962 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/84d09803430347d8a9212e8f657a1962 2024-12-05T05:18:36,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/84d09803430347d8a9212e8f657a1962, entries=100, sequenceid=314, filesize=9.6 K 2024-12-05T05:18:36,981 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=93.93 KB/96180 for 41b82809bb37e31af4591859446916bc in 109ms, sequenceid=314, compaction requested=true 2024-12-05T05:18:36,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:36,982 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:36,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:36,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:36,983 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:36,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:36,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T05:18:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:36,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:36,990 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35103 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:36,990 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:36,990 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:36,990 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/7a485868dd32473c88f7d0973e020355, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d74237e97ef844668d9d91642a2ea0c1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6670dd3b4a5d4e3388deb51808e0450c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=34.3 K 2024-12-05T05:18:36,992 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a485868dd32473c88f7d0973e020355, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733375913928 2024-12-05T05:18:36,992 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32663 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:36,992 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:36,992 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:36,992 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c54b0d3722b048728249bc06b8be8c0d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6da0cc56298f4ed8a9b58881cd7ad8b1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/09809278003d4690898ae138544d4c4b] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=31.9 K 2024-12-05T05:18:36,992 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting d74237e97ef844668d9d91642a2ea0c1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733375914648 2024-12-05T05:18:36,993 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c54b0d3722b048728249bc06b8be8c0d, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733375913928 2024-12-05T05:18:36,993 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6670dd3b4a5d4e3388deb51808e0450c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733375916850 2024-12-05T05:18:36,993 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6da0cc56298f4ed8a9b58881cd7ad8b1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733375914648 2024-12-05T05:18:36,994 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 09809278003d4690898ae138544d4c4b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733375916850 2024-12-05T05:18:36,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/76607f2400f44097bfb6337461c0a9e6 is 50, key is test_row_0/A:col10/1733375916972/Put/seqid=0 2024-12-05T05:18:37,006 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:37,007 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/055439fe4f9946c194bf361be6818dbe is 50, key is test_row_0/B:col10/1733375916870/Put/seqid=0 2024-12-05T05:18:37,014 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:37,015 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/d32e3955ee7b4ef9952e3fc98fb96a73 is 50, key is test_row_0/A:col10/1733375916870/Put/seqid=0 2024-12-05T05:18:37,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742344_1520 (size=14741) 2024-12-05T05:18:37,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742345_1521 (size=13051) 2024-12-05T05:18:37,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742346_1522 (size=13051) 2024-12-05T05:18:37,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:37,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375977078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:37,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:37,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:37,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375977193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:37,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-05T05:18:37,313 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:37,314 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:37,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-05T05:18:37,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T05:18:37,317 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:37,318 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:37,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:37,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:37,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375977403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:37,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:37,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T05:18:37,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/76607f2400f44097bfb6337461c0a9e6 2024-12-05T05:18:37,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/6f1ce0f850394df29c89776199d8555f is 50, key is test_row_0/B:col10/1733375916972/Put/seqid=0 2024-12-05T05:18:37,461 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/d32e3955ee7b4ef9952e3fc98fb96a73 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d32e3955ee7b4ef9952e3fc98fb96a73 2024-12-05T05:18:37,461 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/055439fe4f9946c194bf361be6818dbe as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/055439fe4f9946c194bf361be6818dbe 2024-12-05T05:18:37,468 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into 055439fe4f9946c194bf361be6818dbe(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:37,468 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into d32e3955ee7b4ef9952e3fc98fb96a73(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:37,468 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:37,468 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:37,468 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=13, startTime=1733375916982; duration=0sec 2024-12-05T05:18:37,468 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=13, startTime=1733375916981; duration=0sec 2024-12-05T05:18:37,468 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:37,468 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:37,468 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:37,468 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:37,468 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:37,469 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32663 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:37,469 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:37,469 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,469 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/329e6bb3304146a0a252514c34d48850, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1b8c2803f1324f90ac12edaad186f82e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/84d09803430347d8a9212e8f657a1962] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=31.9 K 2024-12-05T05:18:37,470 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 329e6bb3304146a0a252514c34d48850, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733375913928 2024-12-05T05:18:37,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T05:18:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,470 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b8c2803f1324f90ac12edaad186f82e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1733375914648 2024-12-05T05:18:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,471 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 84d09803430347d8a9212e8f657a1962, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733375916850 2024-12-05T05:18:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742347_1523 (size=12301) 2024-12-05T05:18:37,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/6f1ce0f850394df29c89776199d8555f 2024-12-05T05:18:37,482 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:37,483 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/4123acf318fa4fd5b01fb9b16c3c3b6a is 50, key is test_row_0/C:col10/1733375916870/Put/seqid=0 2024-12-05T05:18:37,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/c470adf2b5fa447793dcaab05e83ac46 is 50, key is test_row_0/C:col10/1733375916972/Put/seqid=0 2024-12-05T05:18:37,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742348_1524 (size=13051) 2024-12-05T05:18:37,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742349_1525 (size=12301) 2024-12-05T05:18:37,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T05:18:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T05:18:37,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375977712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:37,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:37,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:37,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T05:18:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,917 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/4123acf318fa4fd5b01fb9b16c3c3b6a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/4123acf318fa4fd5b01fb9b16c3c3b6a 2024-12-05T05:18:37,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/c470adf2b5fa447793dcaab05e83ac46 2024-12-05T05:18:37,927 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into 4123acf318fa4fd5b01fb9b16c3c3b6a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:37,927 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:37,927 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=13, startTime=1733375916983; duration=0sec 2024-12-05T05:18:37,927 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:37,927 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:37,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/76607f2400f44097bfb6337461c0a9e6 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/76607f2400f44097bfb6337461c0a9e6 2024-12-05T05:18:37,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T05:18:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:37,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:37,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/76607f2400f44097bfb6337461c0a9e6, entries=200, sequenceid=332, filesize=14.4 K 2024-12-05T05:18:37,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/6f1ce0f850394df29c89776199d8555f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6f1ce0f850394df29c89776199d8555f 2024-12-05T05:18:37,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6f1ce0f850394df29c89776199d8555f, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T05:18:37,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/c470adf2b5fa447793dcaab05e83ac46 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c470adf2b5fa447793dcaab05e83ac46 2024-12-05T05:18:37,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c470adf2b5fa447793dcaab05e83ac46, entries=150, sequenceid=332, filesize=12.0 K 2024-12-05T05:18:37,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 41b82809bb37e31af4591859446916bc in 956ms, sequenceid=332, compaction requested=false 2024-12-05T05:18:37,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T05:18:38,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-05T05:18:38,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:38,082 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T05:18:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:38,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:38,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/e274a71ac36349a5b3847f68eaa0e54a is 50, key is test_row_0/A:col10/1733375917058/Put/seqid=0 2024-12-05T05:18:38,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742350_1526 (size=12301) 2024-12-05T05:18:38,104 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/e274a71ac36349a5b3847f68eaa0e54a 2024-12-05T05:18:38,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/c9b7fe9e873a427886f8d3552992ca65 is 50, key is test_row_0/B:col10/1733375917058/Put/seqid=0 2024-12-05T05:18:38,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742351_1527 (size=12301) 2024-12-05T05:18:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:38,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375978327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:38,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:38,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375978433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:38,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T05:18:38,523 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/c9b7fe9e873a427886f8d3552992ca65 2024-12-05T05:18:38,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/13f9050bb2a44e7fb824493b7bbfb252 is 50, key is test_row_0/C:col10/1733375917058/Put/seqid=0 2024-12-05T05:18:38,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742352_1528 (size=12301) 2024-12-05T05:18:38,534 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/13f9050bb2a44e7fb824493b7bbfb252 2024-12-05T05:18:38,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/e274a71ac36349a5b3847f68eaa0e54a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/e274a71ac36349a5b3847f68eaa0e54a 2024-12-05T05:18:38,540 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/e274a71ac36349a5b3847f68eaa0e54a, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T05:18:38,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/c9b7fe9e873a427886f8d3552992ca65 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c9b7fe9e873a427886f8d3552992ca65 2024-12-05T05:18:38,544 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c9b7fe9e873a427886f8d3552992ca65, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T05:18:38,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/13f9050bb2a44e7fb824493b7bbfb252 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/13f9050bb2a44e7fb824493b7bbfb252 2024-12-05T05:18:38,548 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/13f9050bb2a44e7fb824493b7bbfb252, entries=150, sequenceid=349, filesize=12.0 K 2024-12-05T05:18:38,548 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 41b82809bb37e31af4591859446916bc in 466ms, sequenceid=349, compaction requested=true 2024-12-05T05:18:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:38,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-05T05:18:38,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-05T05:18:38,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-05T05:18:38,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2320 sec 2024-12-05T05:18:38,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.2380 sec 2024-12-05T05:18:38,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:38,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-05T05:18:38,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:38,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:38,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:38,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:38,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:38,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:38,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/11173782668749718c90525d87f0df17 is 50, key is test_row_0/A:col10/1733375918303/Put/seqid=0 2024-12-05T05:18:38,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742353_1529 (size=14741) 2024-12-05T05:18:38,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/11173782668749718c90525d87f0df17 2024-12-05T05:18:38,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/980cb1a34f3849ccb6f25d0cd228ef7e is 50, key is test_row_0/B:col10/1733375918303/Put/seqid=0 2024-12-05T05:18:38,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742354_1530 (size=12301) 2024-12-05T05:18:38,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375978767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:38,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:38,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36978 deadline: 1733375978793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:38,800 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:38,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,801 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:38,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:37002 deadline: 1733375978793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:38,803 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in 41b82809bb37e31af4591859446916bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:38,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:38,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375978883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:38,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:39,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375979093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:39,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:39,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/980cb1a34f3849ccb6f25d0cd228ef7e 2024-12-05T05:18:39,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/e58907b626544ef2810384540246243c is 50, key is test_row_0/C:col10/1733375918303/Put/seqid=0 2024-12-05T05:18:39,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742355_1531 (size=12301) 2024-12-05T05:18:39,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/e58907b626544ef2810384540246243c 2024-12-05T05:18:39,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/11173782668749718c90525d87f0df17 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/11173782668749718c90525d87f0df17 2024-12-05T05:18:39,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/11173782668749718c90525d87f0df17, entries=200, sequenceid=368, filesize=14.4 K 2024-12-05T05:18:39,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/980cb1a34f3849ccb6f25d0cd228ef7e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/980cb1a34f3849ccb6f25d0cd228ef7e 2024-12-05T05:18:39,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/980cb1a34f3849ccb6f25d0cd228ef7e, entries=150, sequenceid=368, filesize=12.0 K 2024-12-05T05:18:39,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/e58907b626544ef2810384540246243c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e58907b626544ef2810384540246243c 2024-12-05T05:18:39,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e58907b626544ef2810384540246243c, entries=150, sequenceid=368, filesize=12.0 K 2024-12-05T05:18:39,171 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 41b82809bb37e31af4591859446916bc in 508ms, sequenceid=368, compaction requested=true 2024-12-05T05:18:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:39,171 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:39,171 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:39,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:39,172 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:39,172 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:39,172 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:39,172 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:39,172 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,172 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,172 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d32e3955ee7b4ef9952e3fc98fb96a73, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/76607f2400f44097bfb6337461c0a9e6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/e274a71ac36349a5b3847f68eaa0e54a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/11173782668749718c90525d87f0df17] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=53.5 K 2024-12-05T05:18:39,172 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/055439fe4f9946c194bf361be6818dbe, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6f1ce0f850394df29c89776199d8555f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c9b7fe9e873a427886f8d3552992ca65, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/980cb1a34f3849ccb6f25d0cd228ef7e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=48.8 K 2024-12-05T05:18:39,173 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 055439fe4f9946c194bf361be6818dbe, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733375913928 2024-12-05T05:18:39,173 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting d32e3955ee7b4ef9952e3fc98fb96a73, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733375913928 2024-12-05T05:18:39,173 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76607f2400f44097bfb6337461c0a9e6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733375916917 2024-12-05T05:18:39,173 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f1ce0f850394df29c89776199d8555f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733375916917 2024-12-05T05:18:39,173 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e274a71ac36349a5b3847f68eaa0e54a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375917034 2024-12-05T05:18:39,173 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c9b7fe9e873a427886f8d3552992ca65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375917034 2024-12-05T05:18:39,173 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 980cb1a34f3849ccb6f25d0cd228ef7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733375918303 2024-12-05T05:18:39,173 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11173782668749718c90525d87f0df17, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733375918303 2024-12-05T05:18:39,180 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#453 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:39,180 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/38a9024671f543a1b792447c47c49c63 is 50, key is test_row_0/A:col10/1733375918303/Put/seqid=0 2024-12-05T05:18:39,182 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#454 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:39,183 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/e3ef4fe67be84e66bd3d17eb3d214952 is 50, key is test_row_0/B:col10/1733375918303/Put/seqid=0 2024-12-05T05:18:39,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742357_1533 (size=13187) 2024-12-05T05:18:39,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742356_1532 (size=13187) 2024-12-05T05:18:39,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-05T05:18:39,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:39,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:39,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:39,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:39,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:39,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:39,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/70558793246041e8aba7ce0465fb40f5 is 50, key is test_row_0/A:col10/1733375919408/Put/seqid=0 2024-12-05T05:18:39,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742358_1534 (size=12297) 2024-12-05T05:18:39,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/70558793246041e8aba7ce0465fb40f5 2024-12-05T05:18:39,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/a7b450a603cb4306bf2287a7956f8a7d is 50, key is test_row_0/B:col10/1733375919408/Put/seqid=0 2024-12-05T05:18:39,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742359_1535 (size=9857) 2024-12-05T05:18:39,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-05T05:18:39,463 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:39,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-05T05:18:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T05:18:39,466 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:39,467 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:39,467 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:39,518 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T05:18:39,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:39,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375979545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:39,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T05:18:39,604 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/e3ef4fe67be84e66bd3d17eb3d214952 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/e3ef4fe67be84e66bd3d17eb3d214952 2024-12-05T05:18:39,607 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/38a9024671f543a1b792447c47c49c63 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/38a9024671f543a1b792447c47c49c63 2024-12-05T05:18:39,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T05:18:39,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:39,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:39,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:39,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:39,624 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into 38a9024671f543a1b792447c47c49c63(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:39,625 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:39,625 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=12, startTime=1733375919171; duration=0sec 2024-12-05T05:18:39,625 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:39,625 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:39,625 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:39,625 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into e3ef4fe67be84e66bd3d17eb3d214952(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:39,625 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:39,625 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=12, startTime=1733375919171; duration=0sec 2024-12-05T05:18:39,625 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:39,625 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:39,627 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:39,627 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:39,627 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,627 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/4123acf318fa4fd5b01fb9b16c3c3b6a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c470adf2b5fa447793dcaab05e83ac46, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/13f9050bb2a44e7fb824493b7bbfb252, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e58907b626544ef2810384540246243c] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=48.8 K 2024-12-05T05:18:39,627 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4123acf318fa4fd5b01fb9b16c3c3b6a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1733375913928 2024-12-05T05:18:39,628 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting c470adf2b5fa447793dcaab05e83ac46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1733375916917 2024-12-05T05:18:39,628 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13f9050bb2a44e7fb824493b7bbfb252, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1733375917034 2024-12-05T05:18:39,628 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e58907b626544ef2810384540246243c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733375918303 2024-12-05T05:18:39,636 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:39,637 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/08f7d9921e534eb3a1e979597cfd9ba0 is 50, key is test_row_0/C:col10/1733375918303/Put/seqid=0 2024-12-05T05:18:39,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742360_1536 (size=13187) 2024-12-05T05:18:39,647 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/08f7d9921e534eb3a1e979597cfd9ba0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/08f7d9921e534eb3a1e979597cfd9ba0 2024-12-05T05:18:39,653 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into 08f7d9921e534eb3a1e979597cfd9ba0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:39,653 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:39,653 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=12, startTime=1733375919171; duration=0sec 2024-12-05T05:18:39,653 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:39,653 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:39,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:39,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375979663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:39,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:39,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T05:18:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:39,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T05:18:39,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/a7b450a603cb4306bf2287a7956f8a7d 2024-12-05T05:18:39,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b76bea09ab284c0092ff6bbc284c616c is 50, key is test_row_0/C:col10/1733375919408/Put/seqid=0 2024-12-05T05:18:39,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742361_1537 (size=9857) 2024-12-05T05:18:39,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375979873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:39,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:39,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:39,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:39,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T05:18:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:39,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:39,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:40,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T05:18:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:40,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:40,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:40,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T05:18:40,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:40,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375980183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:40,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T05:18:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:40,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:40,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b76bea09ab284c0092ff6bbc284c616c 2024-12-05T05:18:40,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/70558793246041e8aba7ce0465fb40f5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/70558793246041e8aba7ce0465fb40f5 2024-12-05T05:18:40,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/70558793246041e8aba7ce0465fb40f5, entries=150, sequenceid=382, filesize=12.0 K 2024-12-05T05:18:40,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/a7b450a603cb4306bf2287a7956f8a7d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a7b450a603cb4306bf2287a7956f8a7d 2024-12-05T05:18:40,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a7b450a603cb4306bf2287a7956f8a7d, entries=100, sequenceid=382, filesize=9.6 K 2024-12-05T05:18:40,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b76bea09ab284c0092ff6bbc284c616c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b76bea09ab284c0092ff6bbc284c616c 2024-12-05T05:18:40,295 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b76bea09ab284c0092ff6bbc284c616c, entries=100, sequenceid=382, filesize=9.6 K 2024-12-05T05:18:40,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 41b82809bb37e31af4591859446916bc in 887ms, sequenceid=382, compaction requested=false 2024-12-05T05:18:40,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:40,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-05T05:18:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:40,391 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-05T05:18:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:40,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:40,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b225e03709974feeac9d83639d349857 is 50, key is test_row_0/A:col10/1733375919536/Put/seqid=0 2024-12-05T05:18:40,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742362_1538 (size=12301) 2024-12-05T05:18:40,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T05:18:40,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:40,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:40,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375980693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,701 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:40,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375980697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:40,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:40,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375980701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:40,816 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b225e03709974feeac9d83639d349857 2024-12-05T05:18:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375980813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:40,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375980813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:40,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375980813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:40,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:40,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/293adc05e41b49acbc36aca7b2b67d6d is 50, key is test_row_0/B:col10/1733375919536/Put/seqid=0 2024-12-05T05:18:40,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742363_1539 (size=12301) 2024-12-05T05:18:41,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375981023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375981032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375981032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,230 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/293adc05e41b49acbc36aca7b2b67d6d 2024-12-05T05:18:41,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b96ec38397384e59baf14fcd25f06f0a is 50, key is test_row_0/C:col10/1733375919536/Put/seqid=0 2024-12-05T05:18:41,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742364_1540 (size=12301) 2024-12-05T05:18:41,238 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b96ec38397384e59baf14fcd25f06f0a 2024-12-05T05:18:41,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/b225e03709974feeac9d83639d349857 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b225e03709974feeac9d83639d349857 2024-12-05T05:18:41,250 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b225e03709974feeac9d83639d349857, entries=150, sequenceid=403, filesize=12.0 K 2024-12-05T05:18:41,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/293adc05e41b49acbc36aca7b2b67d6d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/293adc05e41b49acbc36aca7b2b67d6d 2024-12-05T05:18:41,253 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/293adc05e41b49acbc36aca7b2b67d6d, entries=150, sequenceid=403, filesize=12.0 K 2024-12-05T05:18:41,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b96ec38397384e59baf14fcd25f06f0a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b96ec38397384e59baf14fcd25f06f0a 2024-12-05T05:18:41,257 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b96ec38397384e59baf14fcd25f06f0a, entries=150, sequenceid=403, filesize=12.0 K 2024-12-05T05:18:41,257 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 41b82809bb37e31af4591859446916bc in 866ms, sequenceid=403, compaction requested=true 2024-12-05T05:18:41,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:41,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:41,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-05T05:18:41,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-05T05:18:41,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-05T05:18:41,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7930 sec 2024-12-05T05:18:41,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.7980 sec 2024-12-05T05:18:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:41,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-05T05:18:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:41,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:41,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/8cc4cab5fa06419cb896dfe86a7d6e96 is 50, key is test_row_0/A:col10/1733375920699/Put/seqid=0 2024-12-05T05:18:41,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742365_1541 (size=14741) 2024-12-05T05:18:41,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/8cc4cab5fa06419cb896dfe86a7d6e96 2024-12-05T05:18:41,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/306649a0935d471c9313dc321e5c1dbf is 50, key is test_row_0/B:col10/1733375920699/Put/seqid=0 2024-12-05T05:18:41,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742366_1542 (size=12301) 2024-12-05T05:18:41,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375981387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375981388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375981389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375981503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375981503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375981503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-05T05:18:41,613 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:41,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:41,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-05T05:18:41,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T05:18:41,615 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:41,615 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:41,615 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:41,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375981713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375981713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375981714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:41,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T05:18:41,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T05:18:41,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:41,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:41,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:41,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/306649a0935d471c9313dc321e5c1dbf 2024-12-05T05:18:41,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/c98cfcfd7c02404187815eaad1af8384 is 50, key is test_row_0/C:col10/1733375920699/Put/seqid=0 2024-12-05T05:18:41,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742367_1543 (size=12301) 2024-12-05T05:18:41,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T05:18:41,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:41,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:41,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:41,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:41,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:41,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:41,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T05:18:42,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:42,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36966 deadline: 1733375982022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:42,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:42,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:42,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36972 deadline: 1733375982032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:42,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:42,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:36958 deadline: 1733375982033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 , the old value is region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:42,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=2 because the exception is null or not the one we care about 2024-12-05T05:18:42,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T05:18:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:42,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:42,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:42,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:42,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/c98cfcfd7c02404187815eaad1af8384 2024-12-05T05:18:42,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/8cc4cab5fa06419cb896dfe86a7d6e96 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8cc4cab5fa06419cb896dfe86a7d6e96 2024-12-05T05:18:42,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8cc4cab5fa06419cb896dfe86a7d6e96, entries=200, sequenceid=418, filesize=14.4 K 2024-12-05T05:18:42,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/306649a0935d471c9313dc321e5c1dbf as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/306649a0935d471c9313dc321e5c1dbf 2024-12-05T05:18:42,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/306649a0935d471c9313dc321e5c1dbf, entries=150, sequenceid=418, filesize=12.0 K 2024-12-05T05:18:42,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/c98cfcfd7c02404187815eaad1af8384 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c98cfcfd7c02404187815eaad1af8384 2024-12-05T05:18:42,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c98cfcfd7c02404187815eaad1af8384, entries=150, sequenceid=418, filesize=12.0 K 2024-12-05T05:18:42,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 41b82809bb37e31af4591859446916bc in 854ms, sequenceid=418, compaction requested=true 2024-12-05T05:18:42,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:42,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:42,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:42,201 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:42,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:42,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:42,201 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:42,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 41b82809bb37e31af4591859446916bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:42,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:42,205 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:42,205 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/B is initiating minor compaction (all files) 2024-12-05T05:18:42,205 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/B in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:42,205 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/e3ef4fe67be84e66bd3d17eb3d214952, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a7b450a603cb4306bf2287a7956f8a7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/293adc05e41b49acbc36aca7b2b67d6d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/306649a0935d471c9313dc321e5c1dbf] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=46.5 K 2024-12-05T05:18:42,205 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52526 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:42,205 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/A is initiating minor compaction (all files) 2024-12-05T05:18:42,205 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/A in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:42,205 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/38a9024671f543a1b792447c47c49c63, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/70558793246041e8aba7ce0465fb40f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b225e03709974feeac9d83639d349857, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8cc4cab5fa06419cb896dfe86a7d6e96] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=51.3 K 2024-12-05T05:18:42,206 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e3ef4fe67be84e66bd3d17eb3d214952, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733375918303 2024-12-05T05:18:42,206 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38a9024671f543a1b792447c47c49c63, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733375918303 2024-12-05T05:18:42,206 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b450a603cb4306bf2287a7956f8a7d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733375918739 2024-12-05T05:18:42,206 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70558793246041e8aba7ce0465fb40f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733375918739 2024-12-05T05:18:42,207 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 293adc05e41b49acbc36aca7b2b67d6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733375919523 2024-12-05T05:18:42,207 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b225e03709974feeac9d83639d349857, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733375919523 2024-12-05T05:18:42,207 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cc4cab5fa06419cb896dfe86a7d6e96, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733375920677 2024-12-05T05:18:42,207 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 306649a0935d471c9313dc321e5c1dbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733375920695 2024-12-05T05:18:42,230 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:42,231 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/f4146f64f76f4964b1c09e2d206b463b is 50, key is test_row_0/B:col10/1733375920699/Put/seqid=0 2024-12-05T05:18:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T05:18:42,243 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#A#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:42,243 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/854f243cdfb741bdb02ac723eadf97de is 50, key is test_row_0/A:col10/1733375920699/Put/seqid=0 2024-12-05T05:18:42,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-05T05:18:42,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:42,244 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-05T05:18:42,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:42,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:42,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:42,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:42,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:42,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:42,252 INFO [Thread-2053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2053. 2024-12-05T05:18:42,252 DEBUG [Thread-2053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:42,252 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:42,253 DEBUG [Thread-2053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,253 INFO [Thread-2055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2055. 2024-12-05T05:18:42,253 DEBUG [Thread-2055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:42,253 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,253 DEBUG [Thread-2055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,253 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:42,254 INFO [Thread-2051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2051. 2024-12-05T05:18:42,254 DEBUG [Thread-2051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:42,254 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,254 DEBUG [Thread-2051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:42,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/66590c90d6124358a894e55638fd8e66 is 50, key is test_row_0/A:col10/1733375921384/Put/seqid=0 2024-12-05T05:18:42,259 INFO [Thread-2049 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2049. 2024-12-05T05:18:42,259 DEBUG [Thread-2049 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:42,259 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,259 DEBUG [Thread-2049 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,259 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:42,262 INFO [Thread-2047 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2047. 2024-12-05T05:18:42,262 DEBUG [Thread-2047 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:42,262 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,262 DEBUG [Thread-2047 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,262 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:42,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742368_1544 (size=13323) 2024-12-05T05:18:42,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742369_1545 (size=13323) 2024-12-05T05:18:42,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742370_1546 (size=12301) 2024-12-05T05:18:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:42,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. as already flushing 2024-12-05T05:18:42,534 INFO [Thread-2042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2042. 2024-12-05T05:18:42,534 DEBUG [Thread-2042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:42,534 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,534 DEBUG [Thread-2042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:42,555 INFO [Thread-2038 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2038. 2024-12-05T05:18:42,555 DEBUG [Thread-2038 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:42,555 INFO [Thread-2044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2044. 2024-12-05T05:18:42,555 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,555 DEBUG [Thread-2044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:42,555 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,555 DEBUG [Thread-2044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,555 DEBUG [Thread-2038 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:42,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:42,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:42,672 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/66590c90d6124358a894e55638fd8e66 2024-12-05T05:18:42,673 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/f4146f64f76f4964b1c09e2d206b463b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f4146f64f76f4964b1c09e2d206b463b 2024-12-05T05:18:42,676 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/854f243cdfb741bdb02ac723eadf97de as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/854f243cdfb741bdb02ac723eadf97de 2024-12-05T05:18:42,677 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/B of 41b82809bb37e31af4591859446916bc into f4146f64f76f4964b1c09e2d206b463b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:42,677 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:42,677 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/B, priority=12, startTime=1733375922201; duration=0sec 2024-12-05T05:18:42,677 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:42,677 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:B 2024-12-05T05:18:42,677 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:18:42,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/ba1222ba7ba34049aa5c362fed12b07b is 50, key is test_row_0/B:col10/1733375921384/Put/seqid=0 2024-12-05T05:18:42,678 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:18:42,678 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): 41b82809bb37e31af4591859446916bc/C is initiating minor compaction (all files) 2024-12-05T05:18:42,678 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 41b82809bb37e31af4591859446916bc/C in TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:42,678 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/08f7d9921e534eb3a1e979597cfd9ba0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b76bea09ab284c0092ff6bbc284c616c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b96ec38397384e59baf14fcd25f06f0a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c98cfcfd7c02404187815eaad1af8384] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp, totalSize=46.5 K 2024-12-05T05:18:42,678 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 08f7d9921e534eb3a1e979597cfd9ba0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1733375918303 2024-12-05T05:18:42,678 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b76bea09ab284c0092ff6bbc284c616c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1733375918739 2024-12-05T05:18:42,679 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b96ec38397384e59baf14fcd25f06f0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1733375919523 2024-12-05T05:18:42,679 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c98cfcfd7c02404187815eaad1af8384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1733375920695 2024-12-05T05:18:42,680 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/A of 41b82809bb37e31af4591859446916bc into 854f243cdfb741bdb02ac723eadf97de(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:42,680 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:42,680 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/A, priority=12, startTime=1733375922200; duration=0sec 2024-12-05T05:18:42,680 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:42,680 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:A 2024-12-05T05:18:42,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742371_1547 (size=12301) 2024-12-05T05:18:42,684 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 41b82809bb37e31af4591859446916bc#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:42,684 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/27e9f93057d040a8bc0591d50b9309d8 is 50, key is test_row_0/C:col10/1733375920699/Put/seqid=0 2024-12-05T05:18:42,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742372_1548 (size=13323) 2024-12-05T05:18:42,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T05:18:43,081 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/ba1222ba7ba34049aa5c362fed12b07b 2024-12-05T05:18:43,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/8a3e8d74b49d46afa727278678e80c46 is 50, key is test_row_0/C:col10/1733375921384/Put/seqid=0 2024-12-05T05:18:43,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742373_1549 (size=12301) 2024-12-05T05:18:43,090 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/27e9f93057d040a8bc0591d50b9309d8 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/27e9f93057d040a8bc0591d50b9309d8 2024-12-05T05:18:43,094 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 41b82809bb37e31af4591859446916bc/C of 41b82809bb37e31af4591859446916bc into 27e9f93057d040a8bc0591d50b9309d8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:43,094 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:43,094 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., storeName=41b82809bb37e31af4591859446916bc/C, priority=12, startTime=1733375922201; duration=0sec 2024-12-05T05:18:43,094 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:43,094 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 41b82809bb37e31af4591859446916bc:C 2024-12-05T05:18:43,490 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=435 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/8a3e8d74b49d46afa727278678e80c46 2024-12-05T05:18:43,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/66590c90d6124358a894e55638fd8e66 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/66590c90d6124358a894e55638fd8e66 2024-12-05T05:18:43,497 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/66590c90d6124358a894e55638fd8e66, entries=150, sequenceid=435, filesize=12.0 K 2024-12-05T05:18:43,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/ba1222ba7ba34049aa5c362fed12b07b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/ba1222ba7ba34049aa5c362fed12b07b 2024-12-05T05:18:43,501 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/ba1222ba7ba34049aa5c362fed12b07b, entries=150, sequenceid=435, filesize=12.0 K 2024-12-05T05:18:43,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/8a3e8d74b49d46afa727278678e80c46 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8a3e8d74b49d46afa727278678e80c46 2024-12-05T05:18:43,505 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8a3e8d74b49d46afa727278678e80c46, entries=150, sequenceid=435, filesize=12.0 K 2024-12-05T05:18:43,505 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 41b82809bb37e31af4591859446916bc in 1261ms, sequenceid=435, compaction requested=false 2024-12-05T05:18:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 41b82809bb37e31af4591859446916bc: 2024-12-05T05:18:43,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:43,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-05T05:18:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-05T05:18:43,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-05T05:18:43,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-12-05T05:18:43,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.8960 sec 2024-12-05T05:18:43,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-05T05:18:43,763 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:48,854 INFO [Thread-2040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2040. 2024-12-05T05:18:48,854 DEBUG [Thread-2040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:48,854 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:48,854 DEBUG [Thread-2040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:48,854 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:48,874 INFO [Thread-2036 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2036. 2024-12-05T05:18:48,874 DEBUG [Thread-2036 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:18:48,874 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:48,874 DEBUG [Thread-2036 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:48,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 193 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1514 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4535 rows 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1527 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4580 rows 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1523 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4562 rows 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1524 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4567 rows 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1527 2024-12-05T05:18:48,874 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4580 rows 2024-12-05T05:18:48,874 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T05:18:48,874 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:18:48,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:48,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:48,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:18:48,875 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T05:18:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:48,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T05:18:48,877 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375928877"}]},"ts":"1733375928877"} 2024-12-05T05:18:48,878 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T05:18:48,878 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T05:18:48,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:18:48,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, UNASSIGN}] 2024-12-05T05:18:48,881 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, UNASSIGN 2024-12-05T05:18:48,881 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=41b82809bb37e31af4591859446916bc, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:48,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, UNASSIGN because future has completed 2024-12-05T05:18:48,883 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:18:48,883 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:18:48,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T05:18:49,035 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing 41b82809bb37e31af4591859446916bc, disabling compactions & flushes 2024-12-05T05:18:49,035 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. after waiting 0 ms 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:49,035 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing 41b82809bb37e31af4591859446916bc 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=A 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=B 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 41b82809bb37e31af4591859446916bc, store=C 2024-12-05T05:18:49,035 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:49,038 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/fa89ec755d144acf8e7f658cf70e3cae is 50, key is test_row_0/A:col10/1733375928853/Put/seqid=0 2024-12-05T05:18:49,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742374_1550 (size=12301) 2024-12-05T05:18:49,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T05:18:49,441 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/fa89ec755d144acf8e7f658cf70e3cae 2024-12-05T05:18:49,446 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/25a778ef69ee484abc40ccc09d7cdb98 is 50, key is test_row_0/B:col10/1733375928853/Put/seqid=0 2024-12-05T05:18:49,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742375_1551 (size=12301) 2024-12-05T05:18:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T05:18:49,850 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/25a778ef69ee484abc40ccc09d7cdb98 2024-12-05T05:18:49,855 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b306a08a323b463e80d1243cf105ad31 is 50, key is test_row_0/C:col10/1733375928853/Put/seqid=0 2024-12-05T05:18:49,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742376_1552 (size=12301) 2024-12-05T05:18:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T05:18:50,259 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b306a08a323b463e80d1243cf105ad31 2024-12-05T05:18:50,262 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/A/fa89ec755d144acf8e7f658cf70e3cae as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/fa89ec755d144acf8e7f658cf70e3cae 2024-12-05T05:18:50,264 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/fa89ec755d144acf8e7f658cf70e3cae, entries=150, sequenceid=446, filesize=12.0 K 2024-12-05T05:18:50,265 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/B/25a778ef69ee484abc40ccc09d7cdb98 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/25a778ef69ee484abc40ccc09d7cdb98 2024-12-05T05:18:50,267 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/25a778ef69ee484abc40ccc09d7cdb98, entries=150, sequenceid=446, filesize=12.0 K 2024-12-05T05:18:50,268 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/.tmp/C/b306a08a323b463e80d1243cf105ad31 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b306a08a323b463e80d1243cf105ad31 2024-12-05T05:18:50,270 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b306a08a323b463e80d1243cf105ad31, entries=150, sequenceid=446, filesize=12.0 K 2024-12-05T05:18:50,271 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 41b82809bb37e31af4591859446916bc in 1236ms, sequenceid=446, compaction requested=true 2024-12-05T05:18:50,271 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8a7530a0c1784220901ba434fe1a82a9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/83033ab833b140d4949e1a297553c852, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b59360551b9c42aea00426be3653c400, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b3ef69ce710b467081c3a1fe1eb603d2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b4aa0db6ef23410295410e602afac5b3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a0d544a1184448d988238041f1658b7a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/5a56214cf07743f6b4601462e004bc95, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/cc5e42e05bfb427e8d30715a84786a2c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6596afd3ad1447778ce1e7eb1986b10b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/9a8edf36460748b191ac145b0aaf27a4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/675d36d061c94e86898952babc6a0be7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a05c9b119e5e476d96b301c6d35f8dc0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a5f4f04d4dde4bea9dd14606da5b39da, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/1ccddf15026c40b1bc096d70b05e0186, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/c3b9619fdbd942db807a948ee82a53f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/2ff3579063a5481982df16e896af6d58, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/ff6bcef8685942968441087866072833, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/3eea3a8547364bfa8c22cf288615e303, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/4c37a508e82f4abe81a71748361d6681, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/7a485868dd32473c88f7d0973e020355, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d74237e97ef844668d9d91642a2ea0c1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d32e3955ee7b4ef9952e3fc98fb96a73, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6670dd3b4a5d4e3388deb51808e0450c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/76607f2400f44097bfb6337461c0a9e6, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/e274a71ac36349a5b3847f68eaa0e54a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/11173782668749718c90525d87f0df17, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/38a9024671f543a1b792447c47c49c63, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/70558793246041e8aba7ce0465fb40f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b225e03709974feeac9d83639d349857, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8cc4cab5fa06419cb896dfe86a7d6e96] to archive 2024-12-05T05:18:50,272 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:18:50,273 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8a7530a0c1784220901ba434fe1a82a9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8a7530a0c1784220901ba434fe1a82a9 2024-12-05T05:18:50,274 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/83033ab833b140d4949e1a297553c852 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/83033ab833b140d4949e1a297553c852 2024-12-05T05:18:50,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b59360551b9c42aea00426be3653c400 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b59360551b9c42aea00426be3653c400 2024-12-05T05:18:50,275 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b3ef69ce710b467081c3a1fe1eb603d2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b3ef69ce710b467081c3a1fe1eb603d2 2024-12-05T05:18:50,276 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b4aa0db6ef23410295410e602afac5b3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b4aa0db6ef23410295410e602afac5b3 2024-12-05T05:18:50,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a0d544a1184448d988238041f1658b7a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a0d544a1184448d988238041f1658b7a 2024-12-05T05:18:50,277 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/5a56214cf07743f6b4601462e004bc95 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/5a56214cf07743f6b4601462e004bc95 2024-12-05T05:18:50,278 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/cc5e42e05bfb427e8d30715a84786a2c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/cc5e42e05bfb427e8d30715a84786a2c 2024-12-05T05:18:50,279 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6596afd3ad1447778ce1e7eb1986b10b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6596afd3ad1447778ce1e7eb1986b10b 2024-12-05T05:18:50,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/9a8edf36460748b191ac145b0aaf27a4 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/9a8edf36460748b191ac145b0aaf27a4 2024-12-05T05:18:50,280 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/675d36d061c94e86898952babc6a0be7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/675d36d061c94e86898952babc6a0be7 2024-12-05T05:18:50,281 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a05c9b119e5e476d96b301c6d35f8dc0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a05c9b119e5e476d96b301c6d35f8dc0 2024-12-05T05:18:50,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a5f4f04d4dde4bea9dd14606da5b39da to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/a5f4f04d4dde4bea9dd14606da5b39da 2024-12-05T05:18:50,282 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/1ccddf15026c40b1bc096d70b05e0186 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/1ccddf15026c40b1bc096d70b05e0186 2024-12-05T05:18:50,283 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/c3b9619fdbd942db807a948ee82a53f5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/c3b9619fdbd942db807a948ee82a53f5 2024-12-05T05:18:50,284 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/2ff3579063a5481982df16e896af6d58 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/2ff3579063a5481982df16e896af6d58 2024-12-05T05:18:50,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/ff6bcef8685942968441087866072833 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/ff6bcef8685942968441087866072833 2024-12-05T05:18:50,285 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/3eea3a8547364bfa8c22cf288615e303 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/3eea3a8547364bfa8c22cf288615e303 2024-12-05T05:18:50,286 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/4c37a508e82f4abe81a71748361d6681 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/4c37a508e82f4abe81a71748361d6681 2024-12-05T05:18:50,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/7a485868dd32473c88f7d0973e020355 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/7a485868dd32473c88f7d0973e020355 2024-12-05T05:18:50,287 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d74237e97ef844668d9d91642a2ea0c1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d74237e97ef844668d9d91642a2ea0c1 2024-12-05T05:18:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d32e3955ee7b4ef9952e3fc98fb96a73 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/d32e3955ee7b4ef9952e3fc98fb96a73 2024-12-05T05:18:50,288 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6670dd3b4a5d4e3388deb51808e0450c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/6670dd3b4a5d4e3388deb51808e0450c 2024-12-05T05:18:50,289 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/76607f2400f44097bfb6337461c0a9e6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/76607f2400f44097bfb6337461c0a9e6 2024-12-05T05:18:50,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/e274a71ac36349a5b3847f68eaa0e54a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/e274a71ac36349a5b3847f68eaa0e54a 2024-12-05T05:18:50,290 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/11173782668749718c90525d87f0df17 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/11173782668749718c90525d87f0df17 2024-12-05T05:18:50,291 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/38a9024671f543a1b792447c47c49c63 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/38a9024671f543a1b792447c47c49c63 2024-12-05T05:18:50,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/70558793246041e8aba7ce0465fb40f5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/70558793246041e8aba7ce0465fb40f5 2024-12-05T05:18:50,292 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b225e03709974feeac9d83639d349857 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/b225e03709974feeac9d83639d349857 2024-12-05T05:18:50,293 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8cc4cab5fa06419cb896dfe86a7d6e96 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/8cc4cab5fa06419cb896dfe86a7d6e96 2024-12-05T05:18:50,294 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/7707a59ce9f447a993583fc7241d2f41, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/33fc6b27327542c4b0d2755a1876d2de, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/d60f86e9d80240f382c2c6b4ef07a6de, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/84a5912fbca544f28183a97a3a9fbbb7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f8512dbae37c4b008ac70b24e44d3e53, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/727781dc5312413ca7a9ef5375c51b60, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a874e0a61f2d4a44b77a79a445ba8bed, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/0d54e577896b4ea59a490edb41882f25, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/cd9a2c6d3fd54e4594564d1ea2d5546b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f64f3d662c3f4a99935a664c04dd3da9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/53888fce32304564a5060ff27de707c7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/87945ab8ae094ad3a5b95ac483d35cf4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c6f18c47ca724619b81add2c0ae020ad, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/b718bdb407ca4a3790ba6679c86fc66a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/549a11cc29414c8dbb1ed67575735877, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9b5d64bf3b104201a4a4d558e7ad3a27, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/8a912f20caa9464c806eafea5ad2ed20, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/5ea1c5bd0faa4a659fc18e19c574423f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c54b0d3722b048728249bc06b8be8c0d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9fbd1c3ce32542bcbfc2b9749de8b157, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6da0cc56298f4ed8a9b58881cd7ad8b1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/055439fe4f9946c194bf361be6818dbe, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/09809278003d4690898ae138544d4c4b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6f1ce0f850394df29c89776199d8555f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c9b7fe9e873a427886f8d3552992ca65, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/e3ef4fe67be84e66bd3d17eb3d214952, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/980cb1a34f3849ccb6f25d0cd228ef7e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a7b450a603cb4306bf2287a7956f8a7d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/293adc05e41b49acbc36aca7b2b67d6d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/306649a0935d471c9313dc321e5c1dbf] to archive 2024-12-05T05:18:50,295 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:18:50,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/7707a59ce9f447a993583fc7241d2f41 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/7707a59ce9f447a993583fc7241d2f41 2024-12-05T05:18:50,296 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/33fc6b27327542c4b0d2755a1876d2de to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/33fc6b27327542c4b0d2755a1876d2de 2024-12-05T05:18:50,297 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/d60f86e9d80240f382c2c6b4ef07a6de to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/d60f86e9d80240f382c2c6b4ef07a6de 2024-12-05T05:18:50,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/84a5912fbca544f28183a97a3a9fbbb7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/84a5912fbca544f28183a97a3a9fbbb7 2024-12-05T05:18:50,298 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f8512dbae37c4b008ac70b24e44d3e53 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f8512dbae37c4b008ac70b24e44d3e53 2024-12-05T05:18:50,299 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/727781dc5312413ca7a9ef5375c51b60 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/727781dc5312413ca7a9ef5375c51b60 2024-12-05T05:18:50,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a874e0a61f2d4a44b77a79a445ba8bed to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a874e0a61f2d4a44b77a79a445ba8bed 2024-12-05T05:18:50,300 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/0d54e577896b4ea59a490edb41882f25 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/0d54e577896b4ea59a490edb41882f25 2024-12-05T05:18:50,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/cd9a2c6d3fd54e4594564d1ea2d5546b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/cd9a2c6d3fd54e4594564d1ea2d5546b 2024-12-05T05:18:50,301 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f64f3d662c3f4a99935a664c04dd3da9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f64f3d662c3f4a99935a664c04dd3da9 2024-12-05T05:18:50,302 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/53888fce32304564a5060ff27de707c7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/53888fce32304564a5060ff27de707c7 2024-12-05T05:18:50,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/87945ab8ae094ad3a5b95ac483d35cf4 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/87945ab8ae094ad3a5b95ac483d35cf4 2024-12-05T05:18:50,303 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c6f18c47ca724619b81add2c0ae020ad to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c6f18c47ca724619b81add2c0ae020ad 2024-12-05T05:18:50,304 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/b718bdb407ca4a3790ba6679c86fc66a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/b718bdb407ca4a3790ba6679c86fc66a 2024-12-05T05:18:50,305 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/549a11cc29414c8dbb1ed67575735877 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/549a11cc29414c8dbb1ed67575735877 2024-12-05T05:18:50,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9b5d64bf3b104201a4a4d558e7ad3a27 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9b5d64bf3b104201a4a4d558e7ad3a27 2024-12-05T05:18:50,306 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/8a912f20caa9464c806eafea5ad2ed20 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/8a912f20caa9464c806eafea5ad2ed20 2024-12-05T05:18:50,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/5ea1c5bd0faa4a659fc18e19c574423f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/5ea1c5bd0faa4a659fc18e19c574423f 2024-12-05T05:18:50,307 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c54b0d3722b048728249bc06b8be8c0d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c54b0d3722b048728249bc06b8be8c0d 2024-12-05T05:18:50,308 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9fbd1c3ce32542bcbfc2b9749de8b157 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/9fbd1c3ce32542bcbfc2b9749de8b157 2024-12-05T05:18:50,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6da0cc56298f4ed8a9b58881cd7ad8b1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6da0cc56298f4ed8a9b58881cd7ad8b1 2024-12-05T05:18:50,309 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/055439fe4f9946c194bf361be6818dbe to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/055439fe4f9946c194bf361be6818dbe 2024-12-05T05:18:50,310 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/09809278003d4690898ae138544d4c4b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/09809278003d4690898ae138544d4c4b 2024-12-05T05:18:50,311 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6f1ce0f850394df29c89776199d8555f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/6f1ce0f850394df29c89776199d8555f 2024-12-05T05:18:50,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c9b7fe9e873a427886f8d3552992ca65 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/c9b7fe9e873a427886f8d3552992ca65 2024-12-05T05:18:50,312 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/e3ef4fe67be84e66bd3d17eb3d214952 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/e3ef4fe67be84e66bd3d17eb3d214952 2024-12-05T05:18:50,313 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/980cb1a34f3849ccb6f25d0cd228ef7e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/980cb1a34f3849ccb6f25d0cd228ef7e 2024-12-05T05:18:50,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a7b450a603cb4306bf2287a7956f8a7d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/a7b450a603cb4306bf2287a7956f8a7d 2024-12-05T05:18:50,314 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/293adc05e41b49acbc36aca7b2b67d6d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/293adc05e41b49acbc36aca7b2b67d6d 2024-12-05T05:18:50,315 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/306649a0935d471c9313dc321e5c1dbf to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/306649a0935d471c9313dc321e5c1dbf 2024-12-05T05:18:50,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a19662f599f24d8d8e57fd41a5bfcd1c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8237070e81664815bc4ca441fd3a7b82, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/15d7acb64e8a4a0d8c6724f7f5f40fa5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1cdf29717d0f4bedad32d0ab600ea899, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/ce18a70431c744fea7cd63512294099a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/f880c07936eb4143b638aef1ceb74bc1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e7cc194d313b492297d94afd397225e2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/2164ffedc7ec4dad97b8421084737057, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/480403018b64477491494609b4efb974, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a73f21ffab434168a5a498a77cba420e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/312a457f8b8241a6be932fe195ce4221, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/dca9652232074a399050b244b3115428, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/719e842cf3ff4ce791bfe8d6bc9b0749, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/438b8891bcb54487875261e6632c5b05, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6b0a972b71724a239dc96fb95153c6e8, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/fa61a0bce3684d60af41b3efec3e1bed, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/99382a0a7ad04744b5c9832c7aa54d7b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6a7585bdfbd34a71a35c4bd1874910e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/329e6bb3304146a0a252514c34d48850, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/00014b3e3a904a37b6e9a43fa7a10ddd, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1b8c2803f1324f90ac12edaad186f82e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/4123acf318fa4fd5b01fb9b16c3c3b6a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/84d09803430347d8a9212e8f657a1962, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c470adf2b5fa447793dcaab05e83ac46, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/13f9050bb2a44e7fb824493b7bbfb252, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/08f7d9921e534eb3a1e979597cfd9ba0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e58907b626544ef2810384540246243c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b76bea09ab284c0092ff6bbc284c616c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b96ec38397384e59baf14fcd25f06f0a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c98cfcfd7c02404187815eaad1af8384] to archive 2024-12-05T05:18:50,316 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:18:50,317 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a19662f599f24d8d8e57fd41a5bfcd1c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a19662f599f24d8d8e57fd41a5bfcd1c 2024-12-05T05:18:50,318 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8237070e81664815bc4ca441fd3a7b82 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8237070e81664815bc4ca441fd3a7b82 2024-12-05T05:18:50,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/15d7acb64e8a4a0d8c6724f7f5f40fa5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/15d7acb64e8a4a0d8c6724f7f5f40fa5 2024-12-05T05:18:50,319 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1cdf29717d0f4bedad32d0ab600ea899 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1cdf29717d0f4bedad32d0ab600ea899 2024-12-05T05:18:50,320 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/ce18a70431c744fea7cd63512294099a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/ce18a70431c744fea7cd63512294099a 2024-12-05T05:18:50,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/f880c07936eb4143b638aef1ceb74bc1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/f880c07936eb4143b638aef1ceb74bc1 2024-12-05T05:18:50,321 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e7cc194d313b492297d94afd397225e2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e7cc194d313b492297d94afd397225e2 2024-12-05T05:18:50,322 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/2164ffedc7ec4dad97b8421084737057 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/2164ffedc7ec4dad97b8421084737057 2024-12-05T05:18:50,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/480403018b64477491494609b4efb974 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/480403018b64477491494609b4efb974 2024-12-05T05:18:50,323 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a73f21ffab434168a5a498a77cba420e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/a73f21ffab434168a5a498a77cba420e 2024-12-05T05:18:50,324 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/312a457f8b8241a6be932fe195ce4221 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/312a457f8b8241a6be932fe195ce4221 2024-12-05T05:18:50,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/dca9652232074a399050b244b3115428 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/dca9652232074a399050b244b3115428 2024-12-05T05:18:50,325 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/719e842cf3ff4ce791bfe8d6bc9b0749 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/719e842cf3ff4ce791bfe8d6bc9b0749 2024-12-05T05:18:50,326 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/438b8891bcb54487875261e6632c5b05 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/438b8891bcb54487875261e6632c5b05 2024-12-05T05:18:50,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6b0a972b71724a239dc96fb95153c6e8 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6b0a972b71724a239dc96fb95153c6e8 2024-12-05T05:18:50,327 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/fa61a0bce3684d60af41b3efec3e1bed to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/fa61a0bce3684d60af41b3efec3e1bed 2024-12-05T05:18:50,328 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/99382a0a7ad04744b5c9832c7aa54d7b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/99382a0a7ad04744b5c9832c7aa54d7b 2024-12-05T05:18:50,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6a7585bdfbd34a71a35c4bd1874910e7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/6a7585bdfbd34a71a35c4bd1874910e7 2024-12-05T05:18:50,329 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/329e6bb3304146a0a252514c34d48850 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/329e6bb3304146a0a252514c34d48850 2024-12-05T05:18:50,330 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/00014b3e3a904a37b6e9a43fa7a10ddd to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/00014b3e3a904a37b6e9a43fa7a10ddd 2024-12-05T05:18:50,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1b8c2803f1324f90ac12edaad186f82e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/1b8c2803f1324f90ac12edaad186f82e 2024-12-05T05:18:50,331 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/4123acf318fa4fd5b01fb9b16c3c3b6a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/4123acf318fa4fd5b01fb9b16c3c3b6a 2024-12-05T05:18:50,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/84d09803430347d8a9212e8f657a1962 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/84d09803430347d8a9212e8f657a1962 2024-12-05T05:18:50,332 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c470adf2b5fa447793dcaab05e83ac46 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c470adf2b5fa447793dcaab05e83ac46 2024-12-05T05:18:50,333 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/13f9050bb2a44e7fb824493b7bbfb252 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/13f9050bb2a44e7fb824493b7bbfb252 2024-12-05T05:18:50,334 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/08f7d9921e534eb3a1e979597cfd9ba0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/08f7d9921e534eb3a1e979597cfd9ba0 2024-12-05T05:18:50,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e58907b626544ef2810384540246243c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/e58907b626544ef2810384540246243c 2024-12-05T05:18:50,335 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b76bea09ab284c0092ff6bbc284c616c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b76bea09ab284c0092ff6bbc284c616c 2024-12-05T05:18:50,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b96ec38397384e59baf14fcd25f06f0a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b96ec38397384e59baf14fcd25f06f0a 2024-12-05T05:18:50,336 DEBUG [StoreCloser-TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c98cfcfd7c02404187815eaad1af8384 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/c98cfcfd7c02404187815eaad1af8384 2024-12-05T05:18:50,339 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/recovered.edits/449.seqid, newMaxSeqId=449, maxSeqId=1 2024-12-05T05:18:50,340 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. 2024-12-05T05:18:50,340 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for 41b82809bb37e31af4591859446916bc: Waiting for close lock at 1733375929035Running coprocessor pre-close hooks at 1733375929035Disabling compacts and flushes for region at 1733375929035Disabling writes for close at 1733375929035Obtaining lock to block concurrent updates at 1733375929035Preparing flush snapshotting stores in 41b82809bb37e31af4591859446916bc at 1733375929035Finished memstore snapshotting TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733375929035Flushing stores of TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc. at 1733375929036 (+1 ms)Flushing 41b82809bb37e31af4591859446916bc/A: creating writer at 1733375929036Flushing 41b82809bb37e31af4591859446916bc/A: appending metadata at 1733375929038 (+2 ms)Flushing 41b82809bb37e31af4591859446916bc/A: closing flushed file at 1733375929038Flushing 41b82809bb37e31af4591859446916bc/B: creating writer at 1733375929444 (+406 ms)Flushing 41b82809bb37e31af4591859446916bc/B: appending metadata at 1733375929446 (+2 ms)Flushing 41b82809bb37e31af4591859446916bc/B: closing flushed file at 1733375929446Flushing 41b82809bb37e31af4591859446916bc/C: creating writer at 1733375929853 (+407 ms)Flushing 41b82809bb37e31af4591859446916bc/C: appending metadata at 1733375929855 (+2 ms)Flushing 41b82809bb37e31af4591859446916bc/C: closing flushed file at 1733375929855Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e56200b: reopening flushed file at 1733375930261 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1724ae36: reopening flushed file at 1733375930264 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7bef4b93: reopening flushed file at 1733375930267 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 41b82809bb37e31af4591859446916bc in 1236ms, sequenceid=446, compaction requested=true at 1733375930271 (+4 ms)Writing region close event to WAL at 1733375930337 (+66 ms)Running coprocessor post-close hooks at 1733375930340 (+3 ms)Closed at 1733375930340 2024-12-05T05:18:50,341 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed 41b82809bb37e31af4591859446916bc 2024-12-05T05:18:50,342 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=41b82809bb37e31af4591859446916bc, regionState=CLOSED 2024-12-05T05:18:50,343 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure 41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:18:50,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-05T05:18:50,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure 41b82809bb37e31af4591859446916bc, server=8edcaf2ecc84,36233,1733375771401 in 1.4610 sec 2024-12-05T05:18:50,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-12-05T05:18:50,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=41b82809bb37e31af4591859446916bc, UNASSIGN in 1.4650 sec 2024-12-05T05:18:50,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-05T05:18:50,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4680 sec 2024-12-05T05:18:50,349 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375930349"}]},"ts":"1733375930349"} 2024-12-05T05:18:50,350 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T05:18:50,350 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T05:18:50,352 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4760 sec 2024-12-05T05:18:50,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:18:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-05T05:18:51,022 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:51,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T05:18:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:51,024 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T05:18:51,025 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:51,027 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc 2024-12-05T05:18:51,028 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/recovered.edits] 2024-12-05T05:18:51,030 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/66590c90d6124358a894e55638fd8e66 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/66590c90d6124358a894e55638fd8e66 2024-12-05T05:18:51,031 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/854f243cdfb741bdb02ac723eadf97de to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/854f243cdfb741bdb02ac723eadf97de 2024-12-05T05:18:51,032 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/fa89ec755d144acf8e7f658cf70e3cae to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/A/fa89ec755d144acf8e7f658cf70e3cae 2024-12-05T05:18:51,033 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/25a778ef69ee484abc40ccc09d7cdb98 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/25a778ef69ee484abc40ccc09d7cdb98 2024-12-05T05:18:51,034 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/ba1222ba7ba34049aa5c362fed12b07b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/ba1222ba7ba34049aa5c362fed12b07b 2024-12-05T05:18:51,035 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f4146f64f76f4964b1c09e2d206b463b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/B/f4146f64f76f4964b1c09e2d206b463b 2024-12-05T05:18:51,036 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/27e9f93057d040a8bc0591d50b9309d8 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/27e9f93057d040a8bc0591d50b9309d8 2024-12-05T05:18:51,037 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8a3e8d74b49d46afa727278678e80c46 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/8a3e8d74b49d46afa727278678e80c46 2024-12-05T05:18:51,038 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b306a08a323b463e80d1243cf105ad31 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/C/b306a08a323b463e80d1243cf105ad31 2024-12-05T05:18:51,039 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/recovered.edits/449.seqid to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc/recovered.edits/449.seqid 2024-12-05T05:18:51,040 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/41b82809bb37e31af4591859446916bc 2024-12-05T05:18:51,040 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T05:18:51,041 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:51,043 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T05:18:51,045 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T05:18:51,046 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:51,046 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T05:18:51,046 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733375931046"}]},"ts":"9223372036854775807"} 2024-12-05T05:18:51,048 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T05:18:51,048 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 41b82809bb37e31af4591859446916bc, NAME => 'TestAcidGuarantees,,1733375900041.41b82809bb37e31af4591859446916bc.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T05:18:51,048 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T05:18:51,048 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733375931048"}]},"ts":"9223372036854775807"} 2024-12-05T05:18:51,049 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T05:18:51,050 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:51,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 27 msec 2024-12-05T05:18:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-05T05:18:51,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T05:18:51,137 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:51,144 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 249), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=589 (was 636), ProcessCount=11 (was 11), AvailableMemoryMB=2722 (was 3378) 2024-12-05T05:18:51,153 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=589, ProcessCount=11, AvailableMemoryMB=2722 2024-12-05T05:18:51,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:18:51,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:18:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:51,156 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T05:18:51,157 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:51,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-12-05T05:18:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T05:18:51,157 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T05:18:51,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742377_1553 (size=1061) 2024-12-05T05:18:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T05:18:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T05:18:51,564 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426 2024-12-05T05:18:51,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742378_1554 (size=53) 2024-12-05T05:18:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T05:18:51,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:18:51,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ae3f93f3f6ee0cd97fcedac40066c066, disabling compactions & flushes 2024-12-05T05:18:51,970 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:51,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:51,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. after waiting 0 ms 2024-12-05T05:18:51,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:51,970 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:51,970 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ae3f93f3f6ee0cd97fcedac40066c066: Waiting for close lock at 1733375931970Disabling compacts and flushes for region at 1733375931970Disabling writes for close at 1733375931970Writing region close event to WAL at 1733375931970Closed at 1733375931970 2024-12-05T05:18:51,971 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T05:18:51,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733375931971"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733375931971"}]},"ts":"1733375931971"} 2024-12-05T05:18:51,972 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-05T05:18:51,973 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T05:18:51,973 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375931973"}]},"ts":"1733375931973"} 2024-12-05T05:18:51,974 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-05T05:18:51,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, ASSIGN}] 2024-12-05T05:18:51,975 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, ASSIGN 2024-12-05T05:18:51,976 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, ASSIGN; state=OFFLINE, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=false 2024-12-05T05:18:52,126 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=ae3f93f3f6ee0cd97fcedac40066c066, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:52,128 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, ASSIGN because future has completed 2024-12-05T05:18:52,128 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:18:52,282 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:52,282 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:18:52,283 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,283 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:18:52,283 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,283 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,284 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,285 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:52,285 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3f93f3f6ee0cd97fcedac40066c066 columnFamilyName A 2024-12-05T05:18:52,285 DEBUG [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:52,285 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(327): Store=ae3f93f3f6ee0cd97fcedac40066c066/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:52,285 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,286 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:52,286 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3f93f3f6ee0cd97fcedac40066c066 columnFamilyName B 2024-12-05T05:18:52,286 DEBUG [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:52,287 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(327): Store=ae3f93f3f6ee0cd97fcedac40066c066/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:52,287 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,287 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:52,288 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3f93f3f6ee0cd97fcedac40066c066 columnFamilyName C 2024-12-05T05:18:52,288 DEBUG [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:52,288 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(327): Store=ae3f93f3f6ee0cd97fcedac40066c066/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:52,288 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:52,288 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,288 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,289 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,289 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,289 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,290 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:18:52,291 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,292 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T05:18:52,292 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened ae3f93f3f6ee0cd97fcedac40066c066; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72804271, jitterRate=0.0848681777715683}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:18:52,292 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T05:18:52,293 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for ae3f93f3f6ee0cd97fcedac40066c066: Running coprocessor pre-open hook at 1733375932283Writing region info on filesystem at 1733375932283Initializing all the Stores at 1733375932283Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375932283Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375932284 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375932284Cleaning up temporary data from old regions at 1733375932289 (+5 ms)Running coprocessor post-open hooks at 1733375932292 (+3 ms)Region opened successfully at 1733375932293 (+1 ms) 2024-12-05T05:18:52,294 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., pid=140, masterSystemTime=1733375932280 2024-12-05T05:18:52,295 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:52,295 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:52,296 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=ae3f93f3f6ee0cd97fcedac40066c066, regionState=OPEN, openSeqNum=2, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:52,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:18:52,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-05T05:18:52,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 in 170 msec 2024-12-05T05:18:52,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-12-05T05:18:52,301 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, ASSIGN in 325 msec 2024-12-05T05:18:52,301 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T05:18:52,301 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375932301"}]},"ts":"1733375932301"} 2024-12-05T05:18:52,302 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-05T05:18:52,303 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T05:18:52,305 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-12-05T05:18:53,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-12-05T05:18:53,303 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:53,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6856303f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:53,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:53,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:53,305 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63388b3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:53,306 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10812482, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:53,307 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:53,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:53,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:53,309 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-05T05:18:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 8edcaf2ecc84,46695,1733375770505 2024-12-05T05:18:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a6ffe91 2024-12-05T05:18:53,310 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T05:18:53,310 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T05:18:53,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-05T05:18:53,311 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T05:18:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-05T05:18:53,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742379_1555 (size=1097) 2024-12-05T05:18:53,721 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-05T05:18:53,721 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-05T05:18:53,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:18:53,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, REOPEN/MOVE}] 2024-12-05T05:18:53,726 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, REOPEN/MOVE 2024-12-05T05:18:53,727 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ae3f93f3f6ee0cd97fcedac40066c066, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:53,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, REOPEN/MOVE because future has completed 2024-12-05T05:18:53,728 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:18:53,728 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:18:53,880 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:53,880 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:18:53,880 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing ae3f93f3f6ee0cd97fcedac40066c066, disabling compactions & flushes 2024-12-05T05:18:53,881 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:53,881 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:53,881 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. after waiting 0 ms 2024-12-05T05:18:53,881 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:53,884 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-05T05:18:53,884 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:53,884 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for ae3f93f3f6ee0cd97fcedac40066c066: Waiting for close lock at 1733375933880Running coprocessor pre-close hooks at 1733375933880Disabling compacts and flushes for region at 1733375933880Disabling writes for close at 1733375933881 (+1 ms)Writing region close event to WAL at 1733375933881Running coprocessor post-close hooks at 1733375933884 (+3 ms)Closed at 1733375933884 2024-12-05T05:18:53,884 WARN [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: ae3f93f3f6ee0cd97fcedac40066c066 to self. 2024-12-05T05:18:53,886 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:53,886 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ae3f93f3f6ee0cd97fcedac40066c066, regionState=CLOSED 2024-12-05T05:18:53,887 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:18:53,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-05T05:18:53,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 in 160 msec 2024-12-05T05:18:53,890 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, REOPEN/MOVE; state=CLOSED, location=8edcaf2ecc84,36233,1733375771401; forceNewPlan=false, retain=true 2024-12-05T05:18:54,040 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ae3f93f3f6ee0cd97fcedac40066c066, regionState=OPENING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, REOPEN/MOVE because future has completed 2024-12-05T05:18:54,042 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:18:54,195 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,195 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} 2024-12-05T05:18:54,195 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,196 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T05:18:54,196 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,196 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,197 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,197 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:54,197 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3f93f3f6ee0cd97fcedac40066c066 columnFamilyName A 2024-12-05T05:18:54,198 DEBUG [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:54,199 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(327): Store=ae3f93f3f6ee0cd97fcedac40066c066/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:54,199 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,199 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:54,199 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3f93f3f6ee0cd97fcedac40066c066 columnFamilyName B 2024-12-05T05:18:54,199 DEBUG [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:54,200 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(327): Store=ae3f93f3f6ee0cd97fcedac40066c066/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:54,200 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,200 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-05T05:18:54,200 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae3f93f3f6ee0cd97fcedac40066c066 columnFamilyName C 2024-12-05T05:18:54,200 DEBUG [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:54,200 INFO [StoreOpener-ae3f93f3f6ee0cd97fcedac40066c066-1 {}] regionserver.HStore(327): Store=ae3f93f3f6ee0cd97fcedac40066c066/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T05:18:54,200 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,200 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,201 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,202 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,202 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,202 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,202 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-05T05:18:54,203 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,204 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened ae3f93f3f6ee0cd97fcedac40066c066; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72798080, jitterRate=0.08477592468261719}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-05T05:18:54,204 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,204 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for ae3f93f3f6ee0cd97fcedac40066c066: Running coprocessor pre-open hook at 1733375934196Writing region info on filesystem at 1733375934196Initializing all the Stores at 1733375934196Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375934196Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375934197 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733375934197Cleaning up temporary data from old regions at 1733375934202 (+5 ms)Running coprocessor post-open hooks at 1733375934204 (+2 ms)Region opened successfully at 1733375934204 2024-12-05T05:18:54,205 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., pid=145, masterSystemTime=1733375934193 2024-12-05T05:18:54,206 DEBUG [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,206 INFO [RS_OPEN_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,207 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=ae3f93f3f6ee0cd97fcedac40066c066, regionState=OPEN, openSeqNum=5, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:18:54,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-12-05T05:18:54,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 in 168 msec 2024-12-05T05:18:54,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-12-05T05:18:54,211 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, REOPEN/MOVE in 485 msec 2024-12-05T05:18:54,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-05T05:18:54,213 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-12-05T05:18:54,215 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-12-05T05:18:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-05T05:18:54,215 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:54,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a327d39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,217 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288187e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,218 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44bf413a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767cb391, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,219 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,219 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee1777c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,220 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5502a140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,221 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f260380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,221 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,221 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,222 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ea22ba4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,223 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd0c870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ffe9ab6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,224 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,224 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5253126b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,226 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e815799, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,227 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@298d8ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,227 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,227 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,227 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,227 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@489af9c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,229 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,229 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae368b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,230 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b32022f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,230 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,230 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,230 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c726251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,231 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,231 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6ba59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4288ceb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,232 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,232 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,232 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64e60f7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,233 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,234 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f470a56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,234 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c523ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,234 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,234 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,235 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f25c262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,236 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49586, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69508468, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,237 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6258c3ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,237 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,237 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,237 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50bbc266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,238 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74ae0497, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,239 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3221ca67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,239 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8edcaf2ecc84,46695,-1 for getting cluster id 2024-12-05T05:18:54,239 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-05T05:18:54,240 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '749228af-e9f8-4a64-bfdc-0b8e6f58fa86' 2024-12-05T05:18:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-05T05:18:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "749228af-e9f8-4a64-bfdc-0b8e6f58fa86" 2024-12-05T05:18:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8c98f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8edcaf2ecc84,46695,-1] 2024-12-05T05:18:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-05T05:18:54,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:18:54,241 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-05T05:18:54,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c54db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T05:18:54,242 DEBUG [Thread-2498 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,242 DEBUG [Thread-2492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,242 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:54,242 DEBUG [Thread-2487 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,242 DEBUG [Thread-2481 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,242 DEBUG [Thread-2490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,243 DEBUG [Thread-2479 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-12-05T05:18:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,244 DEBUG [Thread-2494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T05:18:54,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,244 DEBUG [Thread-2483 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,245 DEBUG [Thread-2485 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-05T05:18:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53560, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8edcaf2ecc84,36233,1733375771401, seqNum=-1] 2024-12-05T05:18:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T05:18:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,248 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,248 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,249 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:54,249 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:54,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T05:18:54,252 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5] 2024-12-05T05:18:54,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:18:54,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:18:54,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:54,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:18:54,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:54,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:18:54,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:54,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375994270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375994269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733375934262, max timestamp=1733375934273Num uniques 859; 2024-12-05T05:18:54,275 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:54,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733375934262, max timestamp=1733375934273Num uniques 859; 2024-12-05T05:18:54,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:54,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733375934262, max timestamp=1733375934273Num uniques 859; 2024-12-05T05:18:54,276 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:18:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bbd49cd641844e63aa2bed6f611261fa_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375934251/Put/seqid=0 2024-12-05T05:18:54,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742380_1556 (size=12154) 2024-12-05T05:18:54,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375994294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,297 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375994295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375994296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T05:18:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375994382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375994382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T05:18:54,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:54,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375994403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375994403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375994404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T05:18:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T05:18:54,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375994592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375994593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375994613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375994613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375994617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,690 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:54,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205bbd49cd641844e63aa2bed6f611261fa_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bbd49cd641844e63aa2bed6f611261fa_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,693 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/01a4cf1ec4d948ee9acfd579f0aaf4da, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:54,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/01a4cf1ec4d948ee9acfd579f0aaf4da is 175, key is test_row_0/A:col10/1733375934251/Put/seqid=0 2024-12-05T05:18:54,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742381_1557 (size=30955) 2024-12-05T05:18:54,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/01a4cf1ec4d948ee9acfd579f0aaf4da 2024-12-05T05:18:54,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T05:18:54,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:54,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:54,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/b3d76038436746b6a66f31da725308ff is 50, key is test_row_0/B:col10/1733375934251/Put/seqid=0 2024-12-05T05:18:54,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742382_1558 (size=12001) 2024-12-05T05:18:54,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/b3d76038436746b6a66f31da725308ff 2024-12-05T05:18:54,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/4449a498234e4cc88dad014fcbbf86f5 is 50, key is test_row_0/C:col10/1733375934251/Put/seqid=0 2024-12-05T05:18:54,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742383_1559 (size=12001) 2024-12-05T05:18:54,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/4449a498234e4cc88dad014fcbbf86f5 2024-12-05T05:18:54,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/01a4cf1ec4d948ee9acfd579f0aaf4da as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/01a4cf1ec4d948ee9acfd579f0aaf4da 2024-12-05T05:18:54,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/01a4cf1ec4d948ee9acfd579f0aaf4da, entries=150, sequenceid=15, filesize=30.2 K 2024-12-05T05:18:54,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/b3d76038436746b6a66f31da725308ff as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b3d76038436746b6a66f31da725308ff 2024-12-05T05:18:54,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b3d76038436746b6a66f31da725308ff, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T05:18:54,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/4449a498234e4cc88dad014fcbbf86f5 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4449a498234e4cc88dad014fcbbf86f5 2024-12-05T05:18:54,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4449a498234e4cc88dad014fcbbf86f5, entries=150, sequenceid=15, filesize=11.7 K 2024-12-05T05:18:54,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ae3f93f3f6ee0cd97fcedac40066c066 in 540ms, sequenceid=15, compaction requested=false 2024-12-05T05:18:54,793 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-05T05:18:54,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:18:54,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-12-05T05:18:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:54,860 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-05T05:18:54,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:18:54,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:54,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:18:54,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:54,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:18:54,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:54,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120509752579ef6b46989827fe226b71eeb0_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375934295/Put/seqid=0 2024-12-05T05:18:54,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742384_1560 (size=12154) 2024-12-05T05:18:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T05:18:54,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375994917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375994918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,920 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,921 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375994922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375994923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:54,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375994933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:54,935 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375995023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375995023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375995232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375995232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:55,272 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120509752579ef6b46989827fe226b71eeb0_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509752579ef6b46989827fe226b71eeb0_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/61ec288a7b3e46e1af0327a9bb6b878d, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:55,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/61ec288a7b3e46e1af0327a9bb6b878d is 175, key is test_row_0/A:col10/1733375934295/Put/seqid=0 2024-12-05T05:18:55,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742385_1561 (size=30955) 2024-12-05T05:18:55,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T05:18:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375995433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375995433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375995442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375995542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375995542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:55,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:55,676 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/61ec288a7b3e46e1af0327a9bb6b878d 2024-12-05T05:18:55,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c is 50, key is test_row_0/B:col10/1733375934295/Put/seqid=0 2024-12-05T05:18:55,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742386_1562 (size=12001) 2024-12-05T05:18:55,856 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T05:18:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:56,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375996053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,054 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375996053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:56,135 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c 2024-12-05T05:18:56,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e3201bcb1e2742cea2afbaa606ea2c23 is 50, key is test_row_0/C:col10/1733375934295/Put/seqid=0 2024-12-05T05:18:56,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742387_1563 (size=12001) 2024-12-05T05:18:56,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T05:18:56,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:56,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375996443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:56,445 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:56,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375996453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:56,454 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:56,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375996462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:56,463 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:56,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:56,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:56,564 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e3201bcb1e2742cea2afbaa606ea2c23 2024-12-05T05:18:56,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/61ec288a7b3e46e1af0327a9bb6b878d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/61ec288a7b3e46e1af0327a9bb6b878d 2024-12-05T05:18:56,570 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/61ec288a7b3e46e1af0327a9bb6b878d, entries=150, sequenceid=50, filesize=30.2 K 2024-12-05T05:18:56,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c 2024-12-05T05:18:56,574 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T05:18:56,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e3201bcb1e2742cea2afbaa606ea2c23 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e3201bcb1e2742cea2afbaa606ea2c23 2024-12-05T05:18:56,579 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e3201bcb1e2742cea2afbaa606ea2c23, entries=150, sequenceid=50, filesize=11.7 K 2024-12-05T05:18:56,579 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for ae3f93f3f6ee0cd97fcedac40066c066 in 1719ms, sequenceid=50, compaction requested=false 2024-12-05T05:18:56,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:18:56,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:56,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-05T05:18:56,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-05T05:18:56,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-05T05:18:56,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3320 sec 2024-12-05T05:18:56,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 2.3410 sec 2024-12-05T05:18:57,064 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-05T05:18:57,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:18:57,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:57,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:18:57,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:57,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:18:57,066 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:57,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:57,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053ffa3c93fa48403b8158ecc6dd6df4d0_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375934905/Put/seqid=0 2024-12-05T05:18:57,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742388_1564 (size=12154) 2024-12-05T05:18:57,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:57,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412053ffa3c93fa48403b8158ecc6dd6df4d0_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053ffa3c93fa48403b8158ecc6dd6df4d0_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:57,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/bf56dd1064c347159d49102047ac9b20, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:57,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/bf56dd1064c347159d49102047ac9b20 is 175, key is test_row_0/A:col10/1733375934905/Put/seqid=0 2024-12-05T05:18:57,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742389_1565 (size=30955) 2024-12-05T05:18:57,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375997114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,116 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,117 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:57,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375997116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,119 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:57,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375997223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:57,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375997223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,225 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375997433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375997433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:57,506 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/bf56dd1064c347159d49102047ac9b20 2024-12-05T05:18:57,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/c973069fde17468daeba0470db43df47 is 50, key is test_row_0/B:col10/1733375934905/Put/seqid=0 2024-12-05T05:18:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742390_1566 (size=12001) 2024-12-05T05:18:57,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375997742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:57,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375997743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:57,744 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:57,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/c973069fde17468daeba0470db43df47 2024-12-05T05:18:57,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/6672698ca8014b7aa08f1af280c28835 is 50, key is test_row_0/C:col10/1733375934905/Put/seqid=0 2024-12-05T05:18:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742391_1567 (size=12001) 2024-12-05T05:18:58,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375998252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375998252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,256 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,342 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/6672698ca8014b7aa08f1af280c28835 2024-12-05T05:18:58,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/bf56dd1064c347159d49102047ac9b20 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bf56dd1064c347159d49102047ac9b20 2024-12-05T05:18:58,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bf56dd1064c347159d49102047ac9b20, entries=150, sequenceid=62, filesize=30.2 K 2024-12-05T05:18:58,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/c973069fde17468daeba0470db43df47 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/c973069fde17468daeba0470db43df47 2024-12-05T05:18:58,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/c973069fde17468daeba0470db43df47, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T05:18:58,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/6672698ca8014b7aa08f1af280c28835 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6672698ca8014b7aa08f1af280c28835 2024-12-05T05:18:58,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6672698ca8014b7aa08f1af280c28835, entries=150, sequenceid=62, filesize=11.7 K 2024-12-05T05:18:58,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ae3f93f3f6ee0cd97fcedac40066c066 in 1294ms, sequenceid=62, compaction requested=true 2024-12-05T05:18:58,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:18:58,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:18:58,358 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:58,358 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:18:58,359 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:58,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:58,359 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:58,359 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/B is initiating minor compaction (all files) 2024-12-05T05:18:58,359 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/B in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,359 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b3d76038436746b6a66f31da725308ff, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/c973069fde17468daeba0470db43df47] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=35.2 K 2024-12-05T05:18:58,360 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b3d76038436746b6a66f31da725308ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733375934251 2024-12-05T05:18:58,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:18:58,360 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:58,360 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/A is initiating minor compaction (all files) 2024-12-05T05:18:58,361 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/A in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,361 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:58,361 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/01a4cf1ec4d948ee9acfd579f0aaf4da, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/61ec288a7b3e46e1af0327a9bb6b878d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bf56dd1064c347159d49102047ac9b20] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=90.7 K 2024-12-05T05:18:58,361 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,361 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/01a4cf1ec4d948ee9acfd579f0aaf4da, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/61ec288a7b3e46e1af0327a9bb6b878d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bf56dd1064c347159d49102047ac9b20] 2024-12-05T05:18:58,361 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01a4cf1ec4d948ee9acfd579f0aaf4da, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733375934251 2024-12-05T05:18:58,361 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a63c136fd3c4c2c88b1ca7a2d1c0e5c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375934287 2024-12-05T05:18:58,361 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61ec288a7b3e46e1af0327a9bb6b878d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375934287 2024-12-05T05:18:58,361 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c973069fde17468daeba0470db43df47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733375934905 2024-12-05T05:18:58,361 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf56dd1064c347159d49102047ac9b20, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733375934905 2024-12-05T05:18:58,368 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:58,369 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:58,370 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/ed9fd7b2d97e4aec8451b9db75c3403d is 50, key is test_row_0/B:col10/1733375934905/Put/seqid=0 2024-12-05T05:18:58,372 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120553b46bc629f545f88dec2f4fc3cb90de_ae3f93f3f6ee0cd97fcedac40066c066 store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:58,374 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120553b46bc629f545f88dec2f4fc3cb90de_ae3f93f3f6ee0cd97fcedac40066c066, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:58,374 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120553b46bc629f545f88dec2f4fc3cb90de_ae3f93f3f6ee0cd97fcedac40066c066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:58,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742392_1568 (size=12104) 2024-12-05T05:18:58,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742393_1569 (size=4469) 2024-12-05T05:18:58,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-05T05:18:58,403 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:18:58,403 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:18:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-12-05T05:18:58,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T05:18:58,405 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:18:58,405 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:18:58,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:18:58,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:58,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T05:18:58,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:18:58,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:58,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:18:58,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:58,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:18:58,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:18:58,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e448283bf0c945ce90f290ab69f68174_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375937115/Put/seqid=0 2024-12-05T05:18:58,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742394_1570 (size=14594) 2024-12-05T05:18:58,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375998478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,482 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375998478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375998481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,485 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T05:18:58,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375998583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375998593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375998593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T05:18:58,794 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#A#compaction#484 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:58,795 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/ed9fd7b2d97e4aec8451b9db75c3403d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ed9fd7b2d97e4aec8451b9db75c3403d 2024-12-05T05:18:58,795 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/4d01da6dad794c1ea5202205162d471e is 175, key is test_row_0/A:col10/1733375934905/Put/seqid=0 2024-12-05T05:18:58,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375998793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742395_1571 (size=31058) 2024-12-05T05:18:58,800 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/B of ae3f93f3f6ee0cd97fcedac40066c066 into ed9fd7b2d97e4aec8451b9db75c3403d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:58,800 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:18:58,800 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/B, priority=13, startTime=1733375938358; duration=0sec 2024-12-05T05:18:58,800 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:18:58,800 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:B 2024-12-05T05:18:58,801 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:18:58,802 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:18:58,802 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/C is initiating minor compaction (all files) 2024-12-05T05:18:58,802 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/C in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,802 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4449a498234e4cc88dad014fcbbf86f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e3201bcb1e2742cea2afbaa606ea2c23, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6672698ca8014b7aa08f1af280c28835] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=35.2 K 2024-12-05T05:18:58,803 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 4449a498234e4cc88dad014fcbbf86f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733375934251 2024-12-05T05:18:58,803 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e3201bcb1e2742cea2afbaa606ea2c23, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1733375934287 2024-12-05T05:18:58,804 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6672698ca8014b7aa08f1af280c28835, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733375934905 2024-12-05T05:18:58,805 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/4d01da6dad794c1ea5202205162d471e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/4d01da6dad794c1ea5202205162d471e 2024-12-05T05:18:58,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375998803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375998803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:58,806 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:58,809 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/A of ae3f93f3f6ee0cd97fcedac40066c066 into 4d01da6dad794c1ea5202205162d471e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:58,809 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:18:58,809 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/A, priority=13, startTime=1733375938358; duration=0sec 2024-12-05T05:18:58,809 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:58,809 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:A 2024-12-05T05:18:58,815 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:18:58,816 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/0f03aac9b2d54526b16eb154ba75c095 is 50, key is test_row_0/C:col10/1733375934905/Put/seqid=0 2024-12-05T05:18:58,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742396_1572 (size=12104) 2024-12-05T05:18:58,845 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/0f03aac9b2d54526b16eb154ba75c095 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0f03aac9b2d54526b16eb154ba75c095 2024-12-05T05:18:58,848 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/C of ae3f93f3f6ee0cd97fcedac40066c066 into 0f03aac9b2d54526b16eb154ba75c095(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:18:58,848 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:18:58,848 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/C, priority=13, startTime=1733375938360; duration=0sec 2024-12-05T05:18:58,848 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:18:58,848 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:C 2024-12-05T05:18:58,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:58,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:58,865 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:58,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205e448283bf0c945ce90f290ab69f68174_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e448283bf0c945ce90f290ab69f68174_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:58,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/e2653b8e38454b4bb30ea6a02e840e81, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:58,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/e2653b8e38454b4bb30ea6a02e840e81 is 175, key is test_row_0/A:col10/1733375937115/Put/seqid=0 2024-12-05T05:18:58,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742397_1573 (size=39549) 2024-12-05T05:18:59,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T05:18:59,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375999103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:59,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:59,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375999113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:59,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375999113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,114 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:59,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:59,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:59,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733375999272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:59,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:59,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733375999272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:59,274 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:59,274 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:18:59,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,274 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:59,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:59,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/e2653b8e38454b4bb30ea6a02e840e81 2024-12-05T05:18:59,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/8f5588f919684494ba97158178858292 is 50, key is test_row_0/B:col10/1733375937115/Put/seqid=0 2024-12-05T05:18:59,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742398_1574 (size=12001) 2024-12-05T05:18:59,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:59,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T05:18:59,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:59,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733375999612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:59,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:59,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733375999622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:59,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:18:59,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733375999623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:18:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,625 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:18:59,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:18:59,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:59,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:18:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:18:59,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/8f5588f919684494ba97158178858292 2024-12-05T05:18:59,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/53abe7698123446f91761336981ad60b is 50, key is test_row_0/C:col10/1733375937115/Put/seqid=0 2024-12-05T05:18:59,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742399_1575 (size=12001) 2024-12-05T05:18:59,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/53abe7698123446f91761336981ad60b 2024-12-05T05:18:59,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/e2653b8e38454b4bb30ea6a02e840e81 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/e2653b8e38454b4bb30ea6a02e840e81 2024-12-05T05:18:59,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/e2653b8e38454b4bb30ea6a02e840e81, entries=200, sequenceid=83, filesize=38.6 K 2024-12-05T05:18:59,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/8f5588f919684494ba97158178858292 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8f5588f919684494ba97158178858292 2024-12-05T05:18:59,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8f5588f919684494ba97158178858292, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T05:18:59,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/53abe7698123446f91761336981ad60b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/53abe7698123446f91761336981ad60b 2024-12-05T05:18:59,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/53abe7698123446f91761336981ad60b, entries=150, sequenceid=83, filesize=11.7 K 2024-12-05T05:18:59,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ae3f93f3f6ee0cd97fcedac40066c066 in 1267ms, sequenceid=83, compaction requested=false 2024-12-05T05:18:59,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:18:59,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-12-05T05:18:59,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:18:59,784 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:18:59,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:18:59,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:59,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:18:59,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:59,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:18:59,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:18:59,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205df7af7588dd2463eb29fe7753f4cc02b_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375938462/Put/seqid=0 2024-12-05T05:18:59,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742400_1576 (size=12154) 2024-12-05T05:18:59,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:18:59,833 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205df7af7588dd2463eb29fe7753f4cc02b_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205df7af7588dd2463eb29fe7753f4cc02b_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:18:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/57e78050473643b89dbb05fd125a65d0, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:18:59,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/57e78050473643b89dbb05fd125a65d0 is 175, key is test_row_0/A:col10/1733375938462/Put/seqid=0 2024-12-05T05:18:59,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742401_1577 (size=30955) 2024-12-05T05:18:59,846 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/57e78050473643b89dbb05fd125a65d0 2024-12-05T05:18:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/5b120eb97cd64c76b093016d8e445157 is 50, key is test_row_0/B:col10/1733375938462/Put/seqid=0 2024-12-05T05:18:59,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742402_1578 (size=12001) 2024-12-05T05:18:59,862 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/5b120eb97cd64c76b093016d8e445157 2024-12-05T05:18:59,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/4a3262617164440cae0ddfe78649d23c is 50, key is test_row_0/C:col10/1733375938462/Put/seqid=0 2024-12-05T05:18:59,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742403_1579 (size=12001) 2024-12-05T05:19:00,275 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/4a3262617164440cae0ddfe78649d23c 2024-12-05T05:19:00,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/57e78050473643b89dbb05fd125a65d0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/57e78050473643b89dbb05fd125a65d0 2024-12-05T05:19:00,281 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/57e78050473643b89dbb05fd125a65d0, entries=150, sequenceid=97, filesize=30.2 K 2024-12-05T05:19:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/5b120eb97cd64c76b093016d8e445157 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/5b120eb97cd64c76b093016d8e445157 2024-12-05T05:19:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,285 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/5b120eb97cd64c76b093016d8e445157, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T05:19:00,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/4a3262617164440cae0ddfe78649d23c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4a3262617164440cae0ddfe78649d23c 2024-12-05T05:19:00,288 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4a3262617164440cae0ddfe78649d23c, entries=150, sequenceid=97, filesize=11.7 K 2024-12-05T05:19:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,289 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for ae3f93f3f6ee0cd97fcedac40066c066 in 505ms, sequenceid=97, compaction requested=true 2024-12-05T05:19:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:00,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-05T05:19:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-12-05T05:19:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-05T05:19:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,292 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8850 sec 2024-12-05T05:19:00,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.8930 sec 2024-12-05T05:19:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-05T05:19:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,554 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:19:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,554 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:19:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-12-05T05:19:00,555 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:19:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T05:19:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,556 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:19:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,556 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:19:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T05:19:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:00,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:19:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:00,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:00,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:00,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:00,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:00,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T05:19:00,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:00,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:00,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:00,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:00,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:00,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120509e8d22ad4494b3eb79d660d5c9f42f1_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375940687/Put/seqid=0 2024-12-05T05:19:00,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742404_1580 (size=21918) 2024-12-05T05:19:00,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375940721, max timestamp=1733375940734Num uniques 859; 2024-12-05T05:19:00,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375940721, max timestamp=1733375940734Num uniques 859; 2024-12-05T05:19:00,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:00,736 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375940721, max timestamp=1733375940734Num uniques 859; 2024-12-05T05:19:00,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:00,737 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:00,737 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:00,741 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120509e8d22ad4494b3eb79d660d5c9f42f1_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509e8d22ad4494b3eb79d660d5c9f42f1_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:00,741 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/2f26c1d270ee4e2cbd385e3abf3ca860, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:00,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/2f26c1d270ee4e2cbd385e3abf3ca860 is 175, key is test_row_0/A:col10/1733375940687/Put/seqid=0 2024-12-05T05:19:00,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742405_1581 (size=65323) 2024-12-05T05:19:00,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:00,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376000769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,775 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733375940760, max timestamp=1733375940773Num uniques 1018; 2024-12-05T05:19:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T05:19:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733375940760, max timestamp=1733375940773Num uniques 1018; 2024-12-05T05:19:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T05:19:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733375940760, max timestamp=1733375940773Num uniques 1018; 2024-12-05T05:19:00,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T05:19:00,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:00,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376000819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,824 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:00,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:00,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376000823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:00,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T05:19:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:00,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:00,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:00,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:00,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T05:19:00,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:00,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376000883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,887 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:00,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:19:00,933 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-05T05:19:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376000933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:00,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376000933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,936 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:00,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T05:19:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376001092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376001144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376001144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/2f26c1d270ee4e2cbd385e3abf3ca860 2024-12-05T05:19:01,156 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/8da81a85ec9e4698973fdafa97550c54 is 50, key is test_row_0/B:col10/1733375940687/Put/seqid=0 2024-12-05T05:19:01,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742406_1582 (size=12001) 2024-12-05T05:19:01,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/8da81a85ec9e4698973fdafa97550c54 2024-12-05T05:19:01,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/be66c9ef1717429cb58c6276b83d527f is 50, key is test_row_0/C:col10/1733375940687/Put/seqid=0 2024-12-05T05:19:01,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T05:19:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T05:19:01,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742407_1583 (size=12001) 2024-12-05T05:19:01,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/be66c9ef1717429cb58c6276b83d527f 2024-12-05T05:19:01,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/2f26c1d270ee4e2cbd385e3abf3ca860 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/2f26c1d270ee4e2cbd385e3abf3ca860 2024-12-05T05:19:01,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/2f26c1d270ee4e2cbd385e3abf3ca860, entries=350, sequenceid=107, filesize=63.8 K 2024-12-05T05:19:01,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/8da81a85ec9e4698973fdafa97550c54 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8da81a85ec9e4698973fdafa97550c54 2024-12-05T05:19:01,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8da81a85ec9e4698973fdafa97550c54, entries=150, sequenceid=107, filesize=11.7 K 2024-12-05T05:19:01,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/be66c9ef1717429cb58c6276b83d527f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/be66c9ef1717429cb58c6276b83d527f 2024-12-05T05:19:01,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/be66c9ef1717429cb58c6276b83d527f, entries=150, sequenceid=107, filesize=11.7 K 2024-12-05T05:19:01,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ae3f93f3f6ee0cd97fcedac40066c066 in 522ms, sequenceid=107, compaction requested=true 2024-12-05T05:19:01,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:01,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:19:01,211 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:19:01,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:01,211 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:19:01,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:19:01,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:01,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:19:01,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:01,212 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166885 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:19:01,212 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:19:01,212 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/B is initiating minor compaction (all files) 2024-12-05T05:19:01,212 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/A is initiating minor compaction (all files) 2024-12-05T05:19:01,212 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/A in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,212 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/B in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,212 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/4d01da6dad794c1ea5202205162d471e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/e2653b8e38454b4bb30ea6a02e840e81, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/57e78050473643b89dbb05fd125a65d0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/2f26c1d270ee4e2cbd385e3abf3ca860] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=163.0 K 2024-12-05T05:19:01,212 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ed9fd7b2d97e4aec8451b9db75c3403d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8f5588f919684494ba97158178858292, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/5b120eb97cd64c76b093016d8e445157, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8da81a85ec9e4698973fdafa97550c54] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=47.0 K 2024-12-05T05:19:01,212 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,212 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/4d01da6dad794c1ea5202205162d471e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/e2653b8e38454b4bb30ea6a02e840e81, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/57e78050473643b89dbb05fd125a65d0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/2f26c1d270ee4e2cbd385e3abf3ca860] 2024-12-05T05:19:01,212 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ed9fd7b2d97e4aec8451b9db75c3403d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733375934905 2024-12-05T05:19:01,212 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f5588f919684494ba97158178858292, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375937111 2024-12-05T05:19:01,213 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d01da6dad794c1ea5202205162d471e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733375934905 2024-12-05T05:19:01,213 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b120eb97cd64c76b093016d8e445157, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733375938462 2024-12-05T05:19:01,213 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da81a85ec9e4698973fdafa97550c54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733375940674 2024-12-05T05:19:01,213 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2653b8e38454b4bb30ea6a02e840e81, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375937105 2024-12-05T05:19:01,213 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e78050473643b89dbb05fd125a65d0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733375938462 2024-12-05T05:19:01,214 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f26c1d270ee4e2cbd385e3abf3ca860, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733375940657 2024-12-05T05:19:01,227 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:01,227 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:01,228 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/b9e0dd70013743e083e7c60ff3956678 is 50, key is test_row_0/B:col10/1733375940687/Put/seqid=0 2024-12-05T05:19:01,230 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120539c3f73a663e4ffa94343a104e560ae8_ae3f93f3f6ee0cd97fcedac40066c066 store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:01,232 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120539c3f73a663e4ffa94343a104e560ae8_ae3f93f3f6ee0cd97fcedac40066c066, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:01,232 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120539c3f73a663e4ffa94343a104e560ae8_ae3f93f3f6ee0cd97fcedac40066c066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:01,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742409_1585 (size=4469) 2024-12-05T05:19:01,258 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#A#compaction#496 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:01,259 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/181805823d1645ffb0764e298c51318c is 175, key is test_row_0/A:col10/1733375940687/Put/seqid=0 2024-12-05T05:19:01,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742410_1586 (size=31195) 2024-12-05T05:19:01,277 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/181805823d1645ffb0764e298c51318c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/181805823d1645ffb0764e298c51318c 2024-12-05T05:19:01,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742408_1584 (size=12241) 2024-12-05T05:19:01,283 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/A of ae3f93f3f6ee0cd97fcedac40066c066 into 181805823d1645ffb0764e298c51318c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:01,283 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:01,283 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/A, priority=12, startTime=1733375941211; duration=0sec 2024-12-05T05:19:01,283 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:01,283 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:A 2024-12-05T05:19:01,283 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:19:01,287 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:19:01,287 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/C is initiating minor compaction (all files) 2024-12-05T05:19:01,287 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/C in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,287 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0f03aac9b2d54526b16eb154ba75c095, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/53abe7698123446f91761336981ad60b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4a3262617164440cae0ddfe78649d23c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/be66c9ef1717429cb58c6276b83d527f] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=47.0 K 2024-12-05T05:19:01,288 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f03aac9b2d54526b16eb154ba75c095, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1733375934905 2024-12-05T05:19:01,289 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53abe7698123446f91761336981ad60b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733375937111 2024-12-05T05:19:01,289 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a3262617164440cae0ddfe78649d23c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733375938462 2024-12-05T05:19:01,290 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting be66c9ef1717429cb58c6276b83d527f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733375940674 2024-12-05T05:19:01,292 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/b9e0dd70013743e083e7c60ff3956678 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b9e0dd70013743e083e7c60ff3956678 2024-12-05T05:19:01,298 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/B of ae3f93f3f6ee0cd97fcedac40066c066 into b9e0dd70013743e083e7c60ff3956678(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:01,298 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:01,298 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/B, priority=12, startTime=1733375941211; duration=0sec 2024-12-05T05:19:01,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-12-05T05:19:01,298 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:01,298 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:B 2024-12-05T05:19:01,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:01,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:01,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:01,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:01,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:01,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:01,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120572b9f871ed9c4603ac5671acab26f829_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375940800/Put/seqid=0 2024-12-05T05:19:01,311 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:01,311 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/a7c8bd4aa6364ec4b6f164c03bd9ef67 is 50, key is test_row_0/C:col10/1733375940687/Put/seqid=0 2024-12-05T05:19:01,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742411_1587 (size=14794) 2024-12-05T05:19:01,320 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:01,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742412_1588 (size=12241) 2024-12-05T05:19:01,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T05:19:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,329 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120572b9f871ed9c4603ac5671acab26f829_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120572b9f871ed9c4603ac5671acab26f829_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:01,336 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/b48accf1a8d54784bed34577adf90fea, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:01,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376001331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/b48accf1a8d54784bed34577adf90fea is 175, key is test_row_0/A:col10/1733375940800/Put/seqid=0 2024-12-05T05:19:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376001332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742413_1589 (size=39749) 2024-12-05T05:19:01,350 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=163, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/b48accf1a8d54784bed34577adf90fea 2024-12-05T05:19:01,351 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/a7c8bd4aa6364ec4b6f164c03bd9ef67 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/a7c8bd4aa6364ec4b6f164c03bd9ef67 2024-12-05T05:19:01,361 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/C of ae3f93f3f6ee0cd97fcedac40066c066 into a7c8bd4aa6364ec4b6f164c03bd9ef67(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:01,361 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:01,361 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/C, priority=12, startTime=1733375941211; duration=0sec 2024-12-05T05:19:01,361 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:01,361 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:C 2024-12-05T05:19:01,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/272068a567d04cb2969d8c510e414a60 is 50, key is test_row_0/B:col10/1733375940800/Put/seqid=0 2024-12-05T05:19:01,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376001403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742414_1590 (size=12151) 2024-12-05T05:19:01,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/272068a567d04cb2969d8c510e414a60 2024-12-05T05:19:01,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/ac4460b9540e457eb64ceed27a859bd3 is 50, key is test_row_0/C:col10/1733375940800/Put/seqid=0 2024-12-05T05:19:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376001443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376001443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,445 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376001452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376001453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742415_1591 (size=12151) 2024-12-05T05:19:01,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=163 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/ac4460b9540e457eb64ceed27a859bd3 2024-12-05T05:19:01,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/b48accf1a8d54784bed34577adf90fea as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/b48accf1a8d54784bed34577adf90fea 2024-12-05T05:19:01,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/b48accf1a8d54784bed34577adf90fea, entries=200, sequenceid=163, filesize=38.8 K 2024-12-05T05:19:01,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/272068a567d04cb2969d8c510e414a60 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/272068a567d04cb2969d8c510e414a60 2024-12-05T05:19:01,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/272068a567d04cb2969d8c510e414a60, entries=150, sequenceid=163, filesize=11.9 K 2024-12-05T05:19:01,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/ac4460b9540e457eb64ceed27a859bd3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/ac4460b9540e457eb64ceed27a859bd3 2024-12-05T05:19:01,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T05:19:01,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:01,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:01,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/ac4460b9540e457eb64ceed27a859bd3, entries=150, sequenceid=163, filesize=11.9 K 2024-12-05T05:19:01,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=46.96 KB/48090 for ae3f93f3f6ee0cd97fcedac40066c066 in 194ms, sequenceid=163, compaction requested=false 2024-12-05T05:19:01,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:01,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-12-05T05:19:01,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:01,637 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:19:01,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:01,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:01,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:01,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:01,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:01,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:01,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c140e858400b48eb99f783f50d8e72a2_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375941308/Put/seqid=0 2024-12-05T05:19:01,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:01,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:01,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T05:19:01,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742416_1592 (size=12304) 2024-12-05T05:19:01,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375941692, max timestamp=1733375941711Num uniques 859; 2024-12-05T05:19:01,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733375941692, max timestamp=1733375941711Num uniques 859; 2024-12-05T05:19:01,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:19:01,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:19:01,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375941692, max timestamp=1733375941711Num uniques 859; 2024-12-05T05:19:01,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:01,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376001759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,766 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376001766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376001875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,880 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376001883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376001913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376001962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376001962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:01,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376002082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376002093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,096 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:02,109 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c140e858400b48eb99f783f50d8e72a2_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c140e858400b48eb99f783f50d8e72a2_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:02,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/6b4f0d2ef0b447f2a3196e5d8f7397e7, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:02,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/6b4f0d2ef0b447f2a3196e5d8f7397e7 is 175, key is test_row_0/A:col10/1733375941308/Put/seqid=0 2024-12-05T05:19:02,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742417_1593 (size=31105) 2024-12-05T05:19:02,131 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/6b4f0d2ef0b447f2a3196e5d8f7397e7 2024-12-05T05:19:02,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/f458acee1da74397b29a34223ca65628 is 50, key is test_row_0/B:col10/1733375941308/Put/seqid=0 2024-12-05T05:19:02,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742418_1594 (size=12151) 2024-12-05T05:19:02,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376002393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376002403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,562 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/f458acee1da74397b29a34223ca65628 2024-12-05T05:19:02,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/f62dab9561f0490b9432b97dda29c18b is 50, key is test_row_0/C:col10/1733375941308/Put/seqid=0 2024-12-05T05:19:02,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742419_1595 (size=12151) 2024-12-05T05:19:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T05:19:02,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376002904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,907 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376002912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376002922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,924 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376002973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,975 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:02,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:02,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376002983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:02,986 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:02,986 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:03,007 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/f62dab9561f0490b9432b97dda29c18b 2024-12-05T05:19:03,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/6b4f0d2ef0b447f2a3196e5d8f7397e7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6b4f0d2ef0b447f2a3196e5d8f7397e7 2024-12-05T05:19:03,014 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6b4f0d2ef0b447f2a3196e5d8f7397e7, entries=150, sequenceid=174, filesize=30.4 K 2024-12-05T05:19:03,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/f458acee1da74397b29a34223ca65628 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/f458acee1da74397b29a34223ca65628 2024-12-05T05:19:03,019 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/f458acee1da74397b29a34223ca65628, entries=150, sequenceid=174, filesize=11.9 K 2024-12-05T05:19:03,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/f62dab9561f0490b9432b97dda29c18b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/f62dab9561f0490b9432b97dda29c18b 2024-12-05T05:19:03,028 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/f62dab9561f0490b9432b97dda29c18b, entries=150, sequenceid=174, filesize=11.9 K 2024-12-05T05:19:03,029 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for ae3f93f3f6ee0cd97fcedac40066c066 in 1391ms, sequenceid=174, compaction requested=true 2024-12-05T05:19:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:03,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-12-05T05:19:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-12-05T05:19:03,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-12-05T05:19:03,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4740 sec 2024-12-05T05:19:03,036 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 2.4810 sec 2024-12-05T05:19:03,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-05T05:19:03,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:03,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:03,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:03,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:03,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:03,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:03,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:03,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205977d3ee77d2245c4bacbc72ae5545d6d_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375941764/Put/seqid=0 2024-12-05T05:19:03,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742420_1596 (size=14794) 2024-12-05T05:19:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:03,960 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376003957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:03,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376003957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:03,960 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:03,961 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376004063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376004063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376004272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376004273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,277 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,334 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:04,337 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205977d3ee77d2245c4bacbc72ae5545d6d_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205977d3ee77d2245c4bacbc72ae5545d6d_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:04,338 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/96f423e911fb453380bdbc4cae82b909, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:04,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/96f423e911fb453380bdbc4cae82b909 is 175, key is test_row_0/A:col10/1733375941764/Put/seqid=0 2024-12-05T05:19:04,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742421_1597 (size=39749) 2024-12-05T05:19:04,341 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/96f423e911fb453380bdbc4cae82b909 2024-12-05T05:19:04,347 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/3fed77b92e144d5b9f3fd97c7b9c1b48 is 50, key is test_row_0/B:col10/1733375941764/Put/seqid=0 2024-12-05T05:19:04,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742422_1598 (size=12151) 2024-12-05T05:19:04,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376004582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376004582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-12-05T05:19:04,713 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:19:04,713 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:19:04,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-05T05:19:04,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T05:19:04,715 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:19:04,716 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:19:04,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:19:04,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/3fed77b92e144d5b9f3fd97c7b9c1b48 2024-12-05T05:19:04,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e2c86f75efe947c997e73a526e34d820 is 50, key is test_row_0/C:col10/1733375941764/Put/seqid=0 2024-12-05T05:19:04,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742423_1599 (size=12151) 2024-12-05T05:19:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T05:19:04,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T05:19:04,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:04,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:04,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:04,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:04,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:04,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376004942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,944 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376004993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376004993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:04,996 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,996 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:04,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:05,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T05:19:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:05,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:05,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T05:19:05,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:05,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376005093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:05,095 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376005093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:05,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:05,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e2c86f75efe947c997e73a526e34d820 2024-12-05T05:19:05,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/96f423e911fb453380bdbc4cae82b909 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/96f423e911fb453380bdbc4cae82b909 2024-12-05T05:19:05,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/96f423e911fb453380bdbc4cae82b909, entries=200, sequenceid=211, filesize=38.8 K 2024-12-05T05:19:05,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/3fed77b92e144d5b9f3fd97c7b9c1b48 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3fed77b92e144d5b9f3fd97c7b9c1b48 2024-12-05T05:19:05,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T05:19:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:05,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3fed77b92e144d5b9f3fd97c7b9c1b48, entries=150, sequenceid=211, filesize=11.9 K 2024-12-05T05:19:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:05,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e2c86f75efe947c997e73a526e34d820 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e2c86f75efe947c997e73a526e34d820 2024-12-05T05:19:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e2c86f75efe947c997e73a526e34d820, entries=150, sequenceid=211, filesize=11.9 K 2024-12-05T05:19:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~395.09 KB/404576, currentSize=46.96 KB/48090 for ae3f93f3f6ee0cd97fcedac40066c066 in 1264ms, sequenceid=211, compaction requested=true 2024-12-05T05:19:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:05,182 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:19:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:19:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:19:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:05,182 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:19:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:19:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,183 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:19:05,183 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141798 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:19:05,183 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/A is initiating minor compaction (all files) 2024-12-05T05:19:05,183 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/B is initiating minor compaction (all files) 2024-12-05T05:19:05,183 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/A in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,183 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/B in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,184 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/181805823d1645ffb0764e298c51318c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/b48accf1a8d54784bed34577adf90fea, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6b4f0d2ef0b447f2a3196e5d8f7397e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/96f423e911fb453380bdbc4cae82b909] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=138.5 K 2024-12-05T05:19:05,184 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b9e0dd70013743e083e7c60ff3956678, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/272068a567d04cb2969d8c510e414a60, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/f458acee1da74397b29a34223ca65628, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3fed77b92e144d5b9f3fd97c7b9c1b48] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=47.6 K 2024-12-05T05:19:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,184 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,184 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/181805823d1645ffb0764e298c51318c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/b48accf1a8d54784bed34577adf90fea, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6b4f0d2ef0b447f2a3196e5d8f7397e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/96f423e911fb453380bdbc4cae82b909] 2024-12-05T05:19:05,184 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting b9e0dd70013743e083e7c60ff3956678, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733375940674 2024-12-05T05:19:05,184 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 181805823d1645ffb0764e298c51318c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733375940674 2024-12-05T05:19:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,184 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting b48accf1a8d54784bed34577adf90fea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733375940800 2024-12-05T05:19:05,184 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 272068a567d04cb2969d8c510e414a60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733375940800 2024-12-05T05:19:05,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,184 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b4f0d2ef0b447f2a3196e5d8f7397e7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733375941308 2024-12-05T05:19:05,184 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f458acee1da74397b29a34223ca65628, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733375941308 2024-12-05T05:19:05,185 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fed77b92e144d5b9f3fd97c7b9c1b48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733375941756 2024-12-05T05:19:05,185 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96f423e911fb453380bdbc4cae82b909, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733375941743 2024-12-05T05:19:05,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,194 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:05,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,195 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#B#compaction#507 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,195 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/6c41c802e0a345788bd2b59ba58ffb7c is 50, key is test_row_0/B:col10/1733375941764/Put/seqid=0 2024-12-05T05:19:05,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,196 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205a9f8dcc54b9a4f668c4f8abf61392cb0_ae3f93f3f6ee0cd97fcedac40066c066 store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:05,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,198 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205a9f8dcc54b9a4f668c4f8abf61392cb0_ae3f93f3f6ee0cd97fcedac40066c066, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:05,198 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a9f8dcc54b9a4f668c4f8abf61392cb0_ae3f93f3f6ee0cd97fcedac40066c066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742424_1600 (size=12527) 2024-12-05T05:19:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,221 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/6c41c802e0a345788bd2b59ba58ffb7c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/6c41c802e0a345788bd2b59ba58ffb7c 2024-12-05T05:19:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,226 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/B of ae3f93f3f6ee0cd97fcedac40066c066 into 6c41c802e0a345788bd2b59ba58ffb7c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:05,226 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:05,226 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/B, priority=12, startTime=1733375945182; duration=0sec 2024-12-05T05:19:05,226 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:05,226 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:B 2024-12-05T05:19:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,227 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-05T05:19:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,228 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-05T05:19:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,228 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/C is initiating minor compaction (all files) 2024-12-05T05:19:05,228 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/C in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,228 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/a7c8bd4aa6364ec4b6f164c03bd9ef67, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/ac4460b9540e457eb64ceed27a859bd3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/f62dab9561f0490b9432b97dda29c18b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e2c86f75efe947c997e73a526e34d820] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=47.6 K 2024-12-05T05:19:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,229 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a7c8bd4aa6364ec4b6f164c03bd9ef67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733375940674 2024-12-05T05:19:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,229 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ac4460b9540e457eb64ceed27a859bd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=163, earliestPutTs=1733375940800 2024-12-05T05:19:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,229 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting f62dab9561f0490b9432b97dda29c18b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1733375941308 2024-12-05T05:19:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,230 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e2c86f75efe947c997e73a526e34d820, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733375941756 2024-12-05T05:19:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742425_1601 (size=4469) 2024-12-05T05:19:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,241 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#A#compaction#508 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,241 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/0b52a32353ac4e2faea536ae975275db is 175, key is test_row_0/A:col10/1733375941764/Put/seqid=0 2024-12-05T05:19:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,244 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#C#compaction#509 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,244 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/48176596775c43b1895c19de17a678fa is 50, key is test_row_0/C:col10/1733375941764/Put/seqid=0 2024-12-05T05:19:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742426_1602 (size=31481) 2024-12-05T05:19:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742427_1603 (size=12527) 2024-12-05T05:19:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,273 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/0b52a32353ac4e2faea536ae975275db as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/0b52a32353ac4e2faea536ae975275db 2024-12-05T05:19:05,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,283 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/A of ae3f93f3f6ee0cd97fcedac40066c066 into 0b52a32353ac4e2faea536ae975275db(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:05,283 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:05,283 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/A, priority=12, startTime=1733375945182; duration=0sec 2024-12-05T05:19:05,283 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:05,283 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:A 2024-12-05T05:19:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-05T05:19:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:05,328 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:19:05,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:05,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205827958018eaa4046aa7664c699304908_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375943936/Put/seqid=0 2024-12-05T05:19:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T05:19:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742428_1604 (size=9814) 2024-12-05T05:19:05,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,369 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205827958018eaa4046aa7664c699304908_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205827958018eaa4046aa7664c699304908_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/41302f8225344511b954c8ba6ce8f2a3, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:05,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/41302f8225344511b954c8ba6ce8f2a3 is 175, key is test_row_0/A:col10/1733375943936/Put/seqid=0 2024-12-05T05:19:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742429_1605 (size=22461) 2024-12-05T05:19:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,672 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/48176596775c43b1895c19de17a678fa as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48176596775c43b1895c19de17a678fa 2024-12-05T05:19:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,677 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/C of ae3f93f3f6ee0cd97fcedac40066c066 into 48176596775c43b1895c19de17a678fa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:05,677 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:05,677 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/C, priority=12, startTime=1733375945182; duration=0sec 2024-12-05T05:19:05,677 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:05,677 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:C 2024-12-05T05:19:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,791 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/41302f8225344511b954c8ba6ce8f2a3 2024-12-05T05:19:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/0f8c10b1385c4e94a1167be2f28861ee is 50, key is test_row_0/B:col10/1733375943936/Put/seqid=0 2024-12-05T05:19:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742430_1606 (size=9757) 2024-12-05T05:19:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,840 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/0f8c10b1385c4e94a1167be2f28861ee 2024-12-05T05:19:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T05:19:05,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e68061b88e0443c195f8b542a0fe25e0 is 50, key is test_row_0/C:col10/1733375943936/Put/seqid=0 2024-12-05T05:19:05,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742431_1607 (size=9757) 2024-12-05T05:19:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:05,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733375946167, max timestamp=1733375946178Num uniques 859; 2024-12-05T05:19:06,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-05T05:19:06,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375946172, max timestamp=1733375946178Num uniques 859; 2024-12-05T05:19:06,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:06,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375946172, max timestamp=1733375946178Num uniques 859; 2024-12-05T05:19:06,184 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:06,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376006249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,254 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376006253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,283 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e68061b88e0443c195f8b542a0fe25e0 2024-12-05T05:19:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/41302f8225344511b954c8ba6ce8f2a3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/41302f8225344511b954c8ba6ce8f2a3 2024-12-05T05:19:06,294 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/41302f8225344511b954c8ba6ce8f2a3, entries=100, sequenceid=223, filesize=21.9 K 2024-12-05T05:19:06,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/0f8c10b1385c4e94a1167be2f28861ee as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/0f8c10b1385c4e94a1167be2f28861ee 2024-12-05T05:19:06,301 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/0f8c10b1385c4e94a1167be2f28861ee, entries=100, sequenceid=223, filesize=9.5 K 2024-12-05T05:19:06,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/e68061b88e0443c195f8b542a0fe25e0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e68061b88e0443c195f8b542a0fe25e0 2024-12-05T05:19:06,308 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e68061b88e0443c195f8b542a0fe25e0, entries=100, sequenceid=223, filesize=9.5 K 2024-12-05T05:19:06,310 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for ae3f93f3f6ee0cd97fcedac40066c066 in 981ms, sequenceid=223, compaction requested=false 2024-12-05T05:19:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:06,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-05T05:19:06,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-05T05:19:06,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-05T05:19:06,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5950 sec 2024-12-05T05:19:06,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.6010 sec 2024-12-05T05:19:06,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375946238, max timestamp=1733375946365Num uniques 1068; 2024-12-05T05:19:06,369 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-05T05:19:06,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375946238, max timestamp=1733375946366Num uniques 1018; 2024-12-05T05:19:06,372 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T05:19:06,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733375946238, max timestamp=1733375946366Num uniques 1018; 2024-12-05T05:19:06,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-05T05:19:06,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=60.38 KB heapSize=165 KB 2024-12-05T05:19:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:06,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:06,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fa161fb27afb4147992b7236553469c6_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375946384/Put/seqid=0 2024-12-05T05:19:06,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742432_1608 (size=12454) 2024-12-05T05:19:06,469 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,473 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205fa161fb27afb4147992b7236553469c6_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fa161fb27afb4147992b7236553469c6_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:06,474 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/fea71c23797547d9bccfef5c7b450029, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:06,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/fea71c23797547d9bccfef5c7b450029 is 175, key is test_row_0/A:col10/1733375946384/Put/seqid=0 2024-12-05T05:19:06,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375946458, max timestamp=1733375946475Num uniques 859; 2024-12-05T05:19:06,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375946458, max timestamp=1733375946475Num uniques 859; 2024-12-05T05:19:06,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:06,478 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:06,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733375946458, max timestamp=1733375946475Num uniques 859; 2024-12-05T05:19:06,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-05T05:19:06,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742433_1609 (size=31255) 2024-12-05T05:19:06,510 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/fea71c23797547d9bccfef5c7b450029 2024-12-05T05:19:06,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/73c9c97f90334ff6924461c5e02785a2 is 50, key is test_row_0/B:col10/1733375946384/Put/seqid=0 2024-12-05T05:19:06,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742434_1610 (size=12301) 2024-12-05T05:19:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376006561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376006563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/73c9c97f90334ff6924461c5e02785a2 2024-12-05T05:19:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/89e7233325c7412782fef5a4cff3d4d4 is 50, key is test_row_0/C:col10/1733375946384/Put/seqid=0 2024-12-05T05:19:06,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742435_1611 (size=12301) 2024-12-05T05:19:06,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/89e7233325c7412782fef5a4cff3d4d4 2024-12-05T05:19:06,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/fea71c23797547d9bccfef5c7b450029 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/fea71c23797547d9bccfef5c7b450029 2024-12-05T05:19:06,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/fea71c23797547d9bccfef5c7b450029, entries=150, sequenceid=267, filesize=30.5 K 2024-12-05T05:19:06,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/73c9c97f90334ff6924461c5e02785a2 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/73c9c97f90334ff6924461c5e02785a2 2024-12-05T05:19:06,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/73c9c97f90334ff6924461c5e02785a2, entries=150, sequenceid=267, filesize=12.0 K 2024-12-05T05:19:06,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/89e7233325c7412782fef5a4cff3d4d4 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/89e7233325c7412782fef5a4cff3d4d4 2024-12-05T05:19:06,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/89e7233325c7412782fef5a4cff3d4d4, entries=150, sequenceid=267, filesize=12.0 K 2024-12-05T05:19:06,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~164.95 KB/168912, currentSize=127.47 KB/130530 for ae3f93f3f6ee0cd97fcedac40066c066 in 243ms, sequenceid=267, compaction requested=true 2024-12-05T05:19:06,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:06,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:19:06,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:06,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:19:06,636 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:06,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:06,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:19:06,636 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:06,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:06,637 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 85197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:06,637 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/A is initiating minor compaction (all files) 2024-12-05T05:19:06,637 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:06,637 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/A in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:06,637 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/B is initiating minor compaction (all files) 2024-12-05T05:19:06,637 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/B in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:06,637 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/0b52a32353ac4e2faea536ae975275db, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/41302f8225344511b954c8ba6ce8f2a3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/fea71c23797547d9bccfef5c7b450029] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=83.2 K 2024-12-05T05:19:06,637 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/6c41c802e0a345788bd2b59ba58ffb7c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/0f8c10b1385c4e94a1167be2f28861ee, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/73c9c97f90334ff6924461c5e02785a2] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=33.8 K 2024-12-05T05:19:06,637 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:06,638 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/0b52a32353ac4e2faea536ae975275db, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/41302f8225344511b954c8ba6ce8f2a3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/fea71c23797547d9bccfef5c7b450029] 2024-12-05T05:19:06,638 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c41c802e0a345788bd2b59ba58ffb7c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733375941756 2024-12-05T05:19:06,638 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b52a32353ac4e2faea536ae975275db, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733375941756 2024-12-05T05:19:06,638 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f8c10b1385c4e94a1167be2f28861ee, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733375943936 2024-12-05T05:19:06,638 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41302f8225344511b954c8ba6ce8f2a3, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733375943936 2024-12-05T05:19:06,638 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 73c9c97f90334ff6924461c5e02785a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733375946384 2024-12-05T05:19:06,639 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting fea71c23797547d9bccfef5c7b450029, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733375946384 2024-12-05T05:19:06,653 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:06,655 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#B#compaction#517 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:06,656 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/ad6ce8945a3b45bea1ab6d87e5974066 is 50, key is test_row_0/B:col10/1733375946384/Put/seqid=0 2024-12-05T05:19:06,657 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412054f2ef0dd1f6742e683d2b3b7f640c7ac_ae3f93f3f6ee0cd97fcedac40066c066 store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:06,659 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412054f2ef0dd1f6742e683d2b3b7f640c7ac_ae3f93f3f6ee0cd97fcedac40066c066, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:06,659 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054f2ef0dd1f6742e683d2b3b7f640c7ac_ae3f93f3f6ee0cd97fcedac40066c066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:06,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742436_1612 (size=12779) 2024-12-05T05:19:06,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:06,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=134.18 KB heapSize=385.45 KB 2024-12-05T05:19:06,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733375946537, max timestamp=1733375946676Num uniques 1018; 2024-12-05T05:19:06,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T05:19:06,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733375946537, max timestamp=1733375946676Num uniques 1018; 2024-12-05T05:19:06,679 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-05T05:19:06,680 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/ad6ce8945a3b45bea1ab6d87e5974066 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ad6ce8945a3b45bea1ab6d87e5974066 2024-12-05T05:19:06,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:06,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:06,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:06,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:06,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:06,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-05T05:19:06,686 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/B of ae3f93f3f6ee0cd97fcedac40066c066 into ad6ce8945a3b45bea1ab6d87e5974066(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:06,686 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:06,686 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/B, priority=13, startTime=1733375946636; duration=0sec 2024-12-05T05:19:06,686 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:06,686 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:B 2024-12-05T05:19:06,686 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:06,688 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:06,688 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/C is initiating minor compaction (all files) 2024-12-05T05:19:06,688 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/C in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:06,688 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48176596775c43b1895c19de17a678fa, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e68061b88e0443c195f8b542a0fe25e0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/89e7233325c7412782fef5a4cff3d4d4] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=33.8 K 2024-12-05T05:19:06,688 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 48176596775c43b1895c19de17a678fa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1733375941756 2024-12-05T05:19:06,689 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting e68061b88e0443c195f8b542a0fe25e0, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1733375943936 2024-12-05T05:19:06,689 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e7233325c7412782fef5a4cff3d4d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733375946384 2024-12-05T05:19:06,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742437_1613 (size=4469) 2024-12-05T05:19:06,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f0d5e3a19f5d4006aab2f6d60aafb30c_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375946675/Put/seqid=0 2024-12-05T05:19:06,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742438_1614 (size=12454) 2024-12-05T05:19:06,710 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#C#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:06,710 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/552d0a7f86a14f969893d8b993ea22b0 is 50, key is test_row_0/C:col10/1733375946384/Put/seqid=0 2024-12-05T05:19:06,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205f0d5e3a19f5d4006aab2f6d60aafb30c_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f0d5e3a19f5d4006aab2f6d60aafb30c_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:06,716 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/36a4b5abfa794ed0955ef8376e6dea46, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:06,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/36a4b5abfa794ed0955ef8376e6dea46 is 175, key is test_row_0/A:col10/1733375946675/Put/seqid=0 2024-12-05T05:19:06,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742439_1615 (size=12779) 2024-12-05T05:19:06,724 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/552d0a7f86a14f969893d8b993ea22b0 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/552d0a7f86a14f969893d8b993ea22b0 2024-12-05T05:19:06,729 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/C of ae3f93f3f6ee0cd97fcedac40066c066 into 552d0a7f86a14f969893d8b993ea22b0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:06,729 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:06,729 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/C, priority=13, startTime=1733375946636; duration=0sec 2024-12-05T05:19:06,729 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:06,729 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:C 2024-12-05T05:19:06,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742440_1616 (size=31255) 2024-12-05T05:19:06,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/36a4b5abfa794ed0955ef8376e6dea46 2024-12-05T05:19:06,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376006733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376006737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/d510519c54ac4261a4f41e5f3fda93b4 is 50, key is test_row_0/B:col10/1733375946675/Put/seqid=0 2024-12-05T05:19:06,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742441_1617 (size=12301) 2024-12-05T05:19:06,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/d510519c54ac4261a4f41e5f3fda93b4 2024-12-05T05:19:06,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/0182af853e544e5499b4ba968bc1e0a9 is 50, key is test_row_0/C:col10/1733375946675/Put/seqid=0 2024-12-05T05:19:06,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742442_1618 (size=12301) 2024-12-05T05:19:06,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/0182af853e544e5499b4ba968bc1e0a9 2024-12-05T05:19:06,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/36a4b5abfa794ed0955ef8376e6dea46 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/36a4b5abfa794ed0955ef8376e6dea46 2024-12-05T05:19:06,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/36a4b5abfa794ed0955ef8376e6dea46, entries=150, sequenceid=305, filesize=30.5 K 2024-12-05T05:19:06,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/d510519c54ac4261a4f41e5f3fda93b4 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/d510519c54ac4261a4f41e5f3fda93b4 2024-12-05T05:19:06,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/d510519c54ac4261a4f41e5f3fda93b4, entries=150, sequenceid=305, filesize=12.0 K 2024-12-05T05:19:06,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/0182af853e544e5499b4ba968bc1e0a9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0182af853e544e5499b4ba968bc1e0a9 2024-12-05T05:19:06,835 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0182af853e544e5499b4ba968bc1e0a9, entries=150, sequenceid=305, filesize=12.0 K 2024-12-05T05:19:06,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~188.91 KB/193440, currentSize=114.05 KB/116790 for ae3f93f3f6ee0cd97fcedac40066c066 in 158ms, sequenceid=305, compaction requested=false 2024-12-05T05:19:06,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:06,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:06,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-05T05:19:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:06,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:06,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a5f8e16497404835820ed1cef7c85de3_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375946721/Put/seqid=0 2024-12-05T05:19:06,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742443_1619 (size=14994) 2024-12-05T05:19:06,863 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:06,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-05T05:19:06,863 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:19:06,863 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:19:06,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-05T05:19:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T05:19:06,865 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:19:06,866 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:19:06,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:19:06,866 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205a5f8e16497404835820ed1cef7c85de3_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a5f8e16497404835820ed1cef7c85de3_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:06,867 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:06,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9 is 175, key is test_row_0/A:col10/1733375946721/Put/seqid=0 2024-12-05T05:19:06,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376006865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742444_1620 (size=39949) 2024-12-05T05:19:06,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376006869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T05:19:06,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376006973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:06,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:06,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376006982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:06,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:06,986 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:07,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,103 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#A#compaction#516 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:07,103 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/bb8e59a713364c54b4c8fb95f567c4a7 is 175, key is test_row_0/A:col10/1733375946384/Put/seqid=0 2024-12-05T05:19:07,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742445_1621 (size=31733) 2024-12-05T05:19:07,111 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/bb8e59a713364c54b4c8fb95f567c4a7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bb8e59a713364c54b4c8fb95f567c4a7 2024-12-05T05:19:07,116 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/A of ae3f93f3f6ee0cd97fcedac40066c066 into bb8e59a713364c54b4c8fb95f567c4a7(size=31.0 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:07,116 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:07,116 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/A, priority=13, startTime=1733375946636; duration=0sec 2024-12-05T05:19:07,116 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:07,116 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:A 2024-12-05T05:19:07,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T05:19:07,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376007183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:07,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:07,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:07,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376007192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:07,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:07,272 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9 2024-12-05T05:19:07,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a4fe5ffca13c497581f692da881fa710 is 50, key is test_row_0/B:col10/1733375946721/Put/seqid=0 2024-12-05T05:19:07,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742446_1622 (size=12301) 2024-12-05T05:19:07,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:07,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:07,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T05:19:07,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376007493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:07,495 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:07,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376007503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:07,505 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:07,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:07,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:07,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a4fe5ffca13c497581f692da881fa710 2024-12-05T05:19:07,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/9519834a24fa43ae8639b87b3c7ae180 is 50, key is test_row_0/C:col10/1733375946721/Put/seqid=0 2024-12-05T05:19:07,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742447_1623 (size=12301) 2024-12-05T05:19:07,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:07,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:07,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:07,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:07,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:07,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:07,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T05:19:08,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:08,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376008003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:08,005 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:08,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:08,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376008015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:08,016 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:08,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/9519834a24fa43ae8639b87b3c7ae180 2024-12-05T05:19:08,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9 2024-12-05T05:19:08,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:08,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:08,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:08,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:08,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:08,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:08,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9, entries=200, sequenceid=328, filesize=39.0 K 2024-12-05T05:19:08,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:08,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a4fe5ffca13c497581f692da881fa710 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a4fe5ffca13c497581f692da881fa710 2024-12-05T05:19:08,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a4fe5ffca13c497581f692da881fa710, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T05:19:08,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/9519834a24fa43ae8639b87b3c7ae180 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9519834a24fa43ae8639b87b3c7ae180 2024-12-05T05:19:08,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9519834a24fa43ae8639b87b3c7ae180, entries=150, sequenceid=328, filesize=12.0 K 2024-12-05T05:19:08,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for ae3f93f3f6ee0cd97fcedac40066c066 in 1279ms, sequenceid=328, compaction requested=true 2024-12-05T05:19:08,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:08,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:19:08,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:08,123 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:08,123 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:08,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:19:08,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:08,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:19:08,123 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:08,124 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:08,124 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:08,124 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/A is initiating minor compaction (all files) 2024-12-05T05:19:08,124 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/B is initiating minor compaction (all files) 2024-12-05T05:19:08,124 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/A in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:08,124 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/B in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:08,124 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bb8e59a713364c54b4c8fb95f567c4a7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/36a4b5abfa794ed0955ef8376e6dea46, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=100.5 K 2024-12-05T05:19:08,124 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ad6ce8945a3b45bea1ab6d87e5974066, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/d510519c54ac4261a4f41e5f3fda93b4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a4fe5ffca13c497581f692da881fa710] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=36.5 K 2024-12-05T05:19:08,124 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:08,124 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bb8e59a713364c54b4c8fb95f567c4a7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/36a4b5abfa794ed0955ef8376e6dea46, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9] 2024-12-05T05:19:08,124 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ad6ce8945a3b45bea1ab6d87e5974066, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733375946384 2024-12-05T05:19:08,125 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting d510519c54ac4261a4f41e5f3fda93b4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733375946550 2024-12-05T05:19:08,125 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb8e59a713364c54b4c8fb95f567c4a7, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733375946384 2024-12-05T05:19:08,125 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a4fe5ffca13c497581f692da881fa710, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733375946721 2024-12-05T05:19:08,126 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36a4b5abfa794ed0955ef8376e6dea46, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733375946550 2024-12-05T05:19:08,127 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd1a0fdf3c6e4dbf88d208bf625fa7d9, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733375946721 2024-12-05T05:19:08,135 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:08,137 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205ecfc928304a5417888362633d486691f_ae3f93f3f6ee0cd97fcedac40066c066 store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:08,138 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#B#compaction#526 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:08,138 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/ffab145383554835881c6ad81a1815a1 is 50, key is test_row_0/B:col10/1733375946721/Put/seqid=0 2024-12-05T05:19:08,139 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205ecfc928304a5417888362633d486691f_ae3f93f3f6ee0cd97fcedac40066c066, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:08,139 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205ecfc928304a5417888362633d486691f_ae3f93f3f6ee0cd97fcedac40066c066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:08,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742449_1625 (size=12881) 2024-12-05T05:19:08,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742448_1624 (size=4469) 2024-12-05T05:19:08,189 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#A#compaction#525 average throughput is 0.45 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:08,190 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/642f79148738494786d87f2bca6ecf8a is 175, key is test_row_0/A:col10/1733375946721/Put/seqid=0 2024-12-05T05:19:08,191 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/ffab145383554835881c6ad81a1815a1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ffab145383554835881c6ad81a1815a1 2024-12-05T05:19:08,197 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/B of ae3f93f3f6ee0cd97fcedac40066c066 into ffab145383554835881c6ad81a1815a1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:08,197 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:08,197 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/B, priority=13, startTime=1733375948123; duration=0sec 2024-12-05T05:19:08,197 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:08,197 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:B 2024-12-05T05:19:08,197 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:08,198 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:08,198 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/C is initiating minor compaction (all files) 2024-12-05T05:19:08,198 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/C in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:08,198 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/552d0a7f86a14f969893d8b993ea22b0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0182af853e544e5499b4ba968bc1e0a9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9519834a24fa43ae8639b87b3c7ae180] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=36.5 K 2024-12-05T05:19:08,198 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 552d0a7f86a14f969893d8b993ea22b0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733375946384 2024-12-05T05:19:08,198 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 0182af853e544e5499b4ba968bc1e0a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=305, earliestPutTs=1733375946550 2024-12-05T05:19:08,199 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 9519834a24fa43ae8639b87b3c7ae180, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733375946721 2024-12-05T05:19:08,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742450_1626 (size=31835) 2024-12-05T05:19:08,211 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/642f79148738494786d87f2bca6ecf8a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/642f79148738494786d87f2bca6ecf8a 2024-12-05T05:19:08,216 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/A of ae3f93f3f6ee0cd97fcedac40066c066 into 642f79148738494786d87f2bca6ecf8a(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:08,216 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:08,216 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/A, priority=13, startTime=1733375948123; duration=0sec 2024-12-05T05:19:08,216 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:08,216 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:A 2024-12-05T05:19:08,218 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#C#compaction#527 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:08,218 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/6a02a20c0b40454a8a714aede4713f2c is 50, key is test_row_0/C:col10/1733375946721/Put/seqid=0 2024-12-05T05:19:08,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742451_1627 (size=12881) 2024-12-05T05:19:08,249 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/6a02a20c0b40454a8a714aede4713f2c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6a02a20c0b40454a8a714aede4713f2c 2024-12-05T05:19:08,255 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/C of ae3f93f3f6ee0cd97fcedac40066c066 into 6a02a20c0b40454a8a714aede4713f2c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:08,255 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:08,255 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/C, priority=13, startTime=1733375948123; duration=0sec 2024-12-05T05:19:08,256 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:08,256 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:C 2024-12-05T05:19:08,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-05T05:19:08,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:08,262 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-05T05:19:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:08,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:08,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120583f426f082e14441ad5b1f996bdc9e5c_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375946867/Put/seqid=0 2024-12-05T05:19:08,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742452_1628 (size=12454) 2024-12-05T05:19:08,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,323 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120583f426f082e14441ad5b1f996bdc9e5c_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120583f426f082e14441ad5b1f996bdc9e5c_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/d5cd230573ee453285eedfa3ca7b995a, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:08,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/d5cd230573ee453285eedfa3ca7b995a is 175, key is test_row_0/A:col10/1733375946867/Put/seqid=0 2024-12-05T05:19:08,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742453_1629 (size=31255) 2024-12-05T05:19:08,766 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/d5cd230573ee453285eedfa3ca7b995a 2024-12-05T05:19:08,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/2d2a4cd8c6714f7b96af5a710c7cdc1f is 50, key is test_row_0/B:col10/1733375946867/Put/seqid=0 2024-12-05T05:19:08,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742454_1630 (size=12301) 2024-12-05T05:19:08,778 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/2d2a4cd8c6714f7b96af5a710c7cdc1f 2024-12-05T05:19:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/1a02e61a551b4083a662bcb423d3b564 is 50, key is test_row_0/C:col10/1733375946867/Put/seqid=0 2024-12-05T05:19:08,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742455_1631 (size=12301) 2024-12-05T05:19:08,817 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/1a02e61a551b4083a662bcb423d3b564 2024-12-05T05:19:08,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/d5cd230573ee453285eedfa3ca7b995a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/d5cd230573ee453285eedfa3ca7b995a 2024-12-05T05:19:08,825 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/d5cd230573ee453285eedfa3ca7b995a, entries=150, sequenceid=343, filesize=30.5 K 2024-12-05T05:19:08,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/2d2a4cd8c6714f7b96af5a710c7cdc1f as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/2d2a4cd8c6714f7b96af5a710c7cdc1f 2024-12-05T05:19:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,829 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/2d2a4cd8c6714f7b96af5a710c7cdc1f, entries=150, sequenceid=343, filesize=12.0 K 2024-12-05T05:19:08,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/1a02e61a551b4083a662bcb423d3b564 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/1a02e61a551b4083a662bcb423d3b564 2024-12-05T05:19:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,836 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/1a02e61a551b4083a662bcb423d3b564, entries=150, sequenceid=343, filesize=12.0 K 2024-12-05T05:19:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,838 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for ae3f93f3f6ee0cd97fcedac40066c066 in 575ms, sequenceid=343, compaction requested=false 2024-12-05T05:19:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-05T05:19:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-05T05:19:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-05T05:19:08,841 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9730 sec 2024-12-05T05:19:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.9780 sec 2024-12-05T05:19:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:08,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-05T05:19:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,014 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:19:09,014 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:19:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-05T05:19:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,016 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:19:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T05:19:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,017 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:19:09,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:19:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:19:09,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:09,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:09,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:09,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:09,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:09,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:09,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:09,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055ff2f9a08ca24d109a390771f464a918_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375949041/Put/seqid=0 2024-12-05T05:19:09,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733375949066, max timestamp=1733375949076Num uniques 859; 2024-12-05T05:19:09,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T05:19:09,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733375949066, max timestamp=1733375949076Num uniques 859; 2024-12-05T05:19:09,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-05T05:19:09,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742456_1632 (size=12454) 2024-12-05T05:19:09,088 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:09,092 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412055ff2f9a08ca24d109a390771f464a918_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ff2f9a08ca24d109a390771f464a918_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:09,094 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/1279a10179a44003bf43e4021682b99d, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:09,094 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/1279a10179a44003bf43e4021682b99d is 175, key is test_row_0/A:col10/1733375949041/Put/seqid=0 2024-12-05T05:19:09,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742457_1633 (size=31255) 2024-12-05T05:19:09,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376009114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,121 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T05:19:09,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376009120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376009121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376009122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376009122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T05:19:09,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:09,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376009223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,228 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376009232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376009232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376009232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376009232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T05:19:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T05:19:09,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376009432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,433 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376009443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376009443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376009443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376009443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T05:19:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:09,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,498 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=353, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/1279a10179a44003bf43e4021682b99d 2024-12-05T05:19:09,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a9777f45c59e4bb5aa5e1c80784d2a29 is 50, key is test_row_0/B:col10/1733375949041/Put/seqid=0 2024-12-05T05:19:09,518 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T05:19:09,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742458_1634 (size=12301) 2024-12-05T05:19:09,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a9777f45c59e4bb5aa5e1c80784d2a29 2024-12-05T05:19:09,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/c818b53cb2e443dd89f5eb01411b0b6a is 50, key is test_row_0/C:col10/1733375949041/Put/seqid=0 2024-12-05T05:19:09,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742459_1635 (size=12301) 2024-12-05T05:19:09,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T05:19:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T05:19:09,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376009742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376009752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376009752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,756 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,756 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376009753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376009753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:09,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T05:19:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T05:19:09,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:09,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/c818b53cb2e443dd89f5eb01411b0b6a 2024-12-05T05:19:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/1279a10179a44003bf43e4021682b99d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/1279a10179a44003bf43e4021682b99d 2024-12-05T05:19:09,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/1279a10179a44003bf43e4021682b99d, entries=150, sequenceid=353, filesize=30.5 K 2024-12-05T05:19:09,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a9777f45c59e4bb5aa5e1c80784d2a29 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a9777f45c59e4bb5aa5e1c80784d2a29 2024-12-05T05:19:09,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a9777f45c59e4bb5aa5e1c80784d2a29, entries=150, sequenceid=353, filesize=12.0 K 2024-12-05T05:19:09,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/c818b53cb2e443dd89f5eb01411b0b6a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c818b53cb2e443dd89f5eb01411b0b6a 2024-12-05T05:19:09,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c818b53cb2e443dd89f5eb01411b0b6a, entries=150, sequenceid=353, filesize=12.0 K 2024-12-05T05:19:09,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for ae3f93f3f6ee0cd97fcedac40066c066 in 920ms, sequenceid=353, compaction requested=true 2024-12-05T05:19:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:19:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:09,966 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:09,966 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:09,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:19:09,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:09,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:19:09,967 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:09,967 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:09,967 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/B is initiating minor compaction (all files) 2024-12-05T05:19:09,967 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/B in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,967 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ffab145383554835881c6ad81a1815a1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/2d2a4cd8c6714f7b96af5a710c7cdc1f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a9777f45c59e4bb5aa5e1c80784d2a29] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=36.6 K 2024-12-05T05:19:09,968 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:09,968 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/A is initiating minor compaction (all files) 2024-12-05T05:19:09,968 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/A in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,968 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/642f79148738494786d87f2bca6ecf8a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/d5cd230573ee453285eedfa3ca7b995a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/1279a10179a44003bf43e4021682b99d] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=92.1 K 2024-12-05T05:19:09,968 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:09,968 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/642f79148738494786d87f2bca6ecf8a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/d5cd230573ee453285eedfa3ca7b995a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/1279a10179a44003bf43e4021682b99d] 2024-12-05T05:19:09,968 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 642f79148738494786d87f2bca6ecf8a, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733375946721 2024-12-05T05:19:09,968 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting ffab145383554835881c6ad81a1815a1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733375946721 2024-12-05T05:19:09,968 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d2a4cd8c6714f7b96af5a710c7cdc1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733375946858 2024-12-05T05:19:09,968 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5cd230573ee453285eedfa3ca7b995a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733375946858 2024-12-05T05:19:09,969 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting a9777f45c59e4bb5aa5e1c80784d2a29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375949032 2024-12-05T05:19:09,969 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1279a10179a44003bf43e4021682b99d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375949032 2024-12-05T05:19:09,978 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#B#compaction#534 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:09,979 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/8ac3db1c7eee43fe945c734770393483 is 50, key is test_row_0/B:col10/1733375949041/Put/seqid=0 2024-12-05T05:19:09,981 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:09,983 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120502f20b2967ea4b46b1153065bbc48f83_ae3f93f3f6ee0cd97fcedac40066c066 store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:09,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742460_1636 (size=12983) 2024-12-05T05:19:09,985 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120502f20b2967ea4b46b1153065bbc48f83_ae3f93f3f6ee0cd97fcedac40066c066, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:09,985 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120502f20b2967ea4b46b1153065bbc48f83_ae3f93f3f6ee0cd97fcedac40066c066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:09,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742461_1637 (size=4469) 2024-12-05T05:19:10,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-05T05:19:10,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:10,087 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=147.60 KB heapSize=423.55 KB 2024-12-05T05:19:10,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:10,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:10,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:10,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:10,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:10,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:10,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205eb0f6a3557f543da9f18caa0991b17af_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375949120/Put/seqid=0 2024-12-05T05:19:10,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742462_1638 (size=12454) 2024-12-05T05:19:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T05:19:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:10,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:10,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376010263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376010265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376010266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,271 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376010271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376010271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376010372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,375 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376010382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376010382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376010382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,385 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,389 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/8ac3db1c7eee43fe945c734770393483 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8ac3db1c7eee43fe945c734770393483 2024-12-05T05:19:10,390 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#A#compaction#535 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:10,390 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/45c4c315cf424e1da8836edc86bf6d01 is 175, key is test_row_0/A:col10/1733375949041/Put/seqid=0 2024-12-05T05:19:10,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742463_1639 (size=31937) 2024-12-05T05:19:10,394 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/B of ae3f93f3f6ee0cd97fcedac40066c066 into 8ac3db1c7eee43fe945c734770393483(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:10,394 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:10,394 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/B, priority=13, startTime=1733375949966; duration=0sec 2024-12-05T05:19:10,394 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:10,394 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:B 2024-12-05T05:19:10,394 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:10,398 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:10,398 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/C is initiating minor compaction (all files) 2024-12-05T05:19:10,398 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/C in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:10,398 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6a02a20c0b40454a8a714aede4713f2c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/1a02e61a551b4083a662bcb423d3b564, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c818b53cb2e443dd89f5eb01411b0b6a] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=36.6 K 2024-12-05T05:19:10,399 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a02a20c0b40454a8a714aede4713f2c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1733375946721 2024-12-05T05:19:10,400 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a02e61a551b4083a662bcb423d3b564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1733375946858 2024-12-05T05:19:10,400 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/45c4c315cf424e1da8836edc86bf6d01 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/45c4c315cf424e1da8836edc86bf6d01 2024-12-05T05:19:10,400 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting c818b53cb2e443dd89f5eb01411b0b6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375949032 2024-12-05T05:19:10,405 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/A of ae3f93f3f6ee0cd97fcedac40066c066 into 45c4c315cf424e1da8836edc86bf6d01(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:10,405 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:10,405 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/A, priority=13, startTime=1733375949966; duration=0sec 2024-12-05T05:19:10,405 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:10,405 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:A 2024-12-05T05:19:10,406 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#C#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:10,407 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/3fa5cad36bd34e948662cc982cdad741 is 50, key is test_row_0/C:col10/1733375949041/Put/seqid=0 2024-12-05T05:19:10,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742464_1640 (size=12983) 2024-12-05T05:19:10,441 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/3fa5cad36bd34e948662cc982cdad741 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3fa5cad36bd34e948662cc982cdad741 2024-12-05T05:19:10,446 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/C of ae3f93f3f6ee0cd97fcedac40066c066 into 3fa5cad36bd34e948662cc982cdad741(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:10,446 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:10,446 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/C, priority=13, startTime=1733375949967; duration=0sec 2024-12-05T05:19:10,446 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:10,446 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:C 2024-12-05T05:19:10,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:10,505 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205eb0f6a3557f543da9f18caa0991b17af_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eb0f6a3557f543da9f18caa0991b17af_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:10,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/f72fc149db9e4a06b5720ee56570db2b, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:10,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/f72fc149db9e4a06b5720ee56570db2b is 175, key is test_row_0/A:col10/1733375949120/Put/seqid=0 2024-12-05T05:19:10,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742465_1641 (size=31255) 2024-12-05T05:19:10,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376010583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376010593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376010593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376010593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,597 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376010892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376010903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376010903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376010903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,922 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=386, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/f72fc149db9e4a06b5720ee56570db2b 2024-12-05T05:19:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:10,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:10,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/32ee0876d9d747d5bfc3e441294440c3 is 50, key is test_row_0/B:col10/1733375949120/Put/seqid=0 2024-12-05T05:19:10,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742466_1642 (size=12301) 2024-12-05T05:19:10,966 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/32ee0876d9d747d5bfc3e441294440c3 2024-12-05T05:19:10,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/48230c08ce3e420a86fbb20a81d9aee3 is 50, key is test_row_0/C:col10/1733375949120/Put/seqid=0 2024-12-05T05:19:10,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742467_1643 (size=12301) 2024-12-05T05:19:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T05:19:11,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376011282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,285 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_1' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,386 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.09 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/48230c08ce3e420a86fbb20a81d9aee3 2024-12-05T05:19:11,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/f72fc149db9e4a06b5720ee56570db2b as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f72fc149db9e4a06b5720ee56570db2b 2024-12-05T05:19:11,393 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f72fc149db9e4a06b5720ee56570db2b, entries=150, sequenceid=386, filesize=30.5 K 2024-12-05T05:19:11,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/32ee0876d9d747d5bfc3e441294440c3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/32ee0876d9d747d5bfc3e441294440c3 2024-12-05T05:19:11,396 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/32ee0876d9d747d5bfc3e441294440c3, entries=150, sequenceid=386, filesize=12.0 K 2024-12-05T05:19:11,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/48230c08ce3e420a86fbb20a81d9aee3 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48230c08ce3e420a86fbb20a81d9aee3 2024-12-05T05:19:11,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,399 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48230c08ce3e420a86fbb20a81d9aee3, entries=150, sequenceid=386, filesize=12.0 K 2024-12-05T05:19:11,399 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~423.50 KB/433664, currentSize=40.25 KB/41220 for ae3f93f3f6ee0cd97fcedac40066c066 in 1312ms, sequenceid=386, compaction requested=false 2024-12-05T05:19:11,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:11,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:11,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-05T05:19:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-05T05:19:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-05T05:19:11,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3840 sec 2024-12-05T05:19:11,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-05T05:19:11,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:11,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:11,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:11,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:11,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:11,404 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:11,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.3890 sec 2024-12-05T05:19:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050b4ed95c5a2e4781864dbc6be4a34556_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375950266/Put/seqid=0 2024-12-05T05:19:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742468_1644 (size=12454) 2024-12-05T05:19:11,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733375951424, max timestamp=1733375951448Num uniques 859; 2024-12-05T05:19:11,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T05:19:11,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733375951424, max timestamp=1733375951448Num uniques 859; 2024-12-05T05:19:11,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T05:19:11,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733375951424, max timestamp=1733375951448Num uniques 859; 2024-12-05T05:19:11,451 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/8edcaf2ecc84:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-05T05:19:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376011468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,472 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376011472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376011472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376011473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,478 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376011582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376011582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376011582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376011583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376011792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376011792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376011792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376011793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:11,797 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:11,815 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:11,818 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412050b4ed95c5a2e4781864dbc6be4a34556_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050b4ed95c5a2e4781864dbc6be4a34556_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:11,818 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/6f52d699e43b48bfbd5130cd5ccd3326, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:11,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/6f52d699e43b48bfbd5130cd5ccd3326 is 175, key is test_row_0/A:col10/1733375950266/Put/seqid=0 2024-12-05T05:19:11,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742469_1645 (size=31255) 2024-12-05T05:19:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376012103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376012103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:12,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376012103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:12,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:12,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376012103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:12,223 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=399, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/6f52d699e43b48bfbd5130cd5ccd3326 2024-12-05T05:19:12,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/3834bdce9d0a4c399b32608768facdb1 is 50, key is test_row_0/B:col10/1733375950266/Put/seqid=0 2024-12-05T05:19:12,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742470_1646 (size=12301) 2024-12-05T05:19:12,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376012613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376012613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:12,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376012613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:12,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376012613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,616 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:12,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/3834bdce9d0a4c399b32608768facdb1 2024-12-05T05:19:12,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/24c808dfa3a34d31bc7f324a9bc6e51e is 50, key is test_row_0/C:col10/1733375950266/Put/seqid=0 2024-12-05T05:19:12,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742471_1647 (size=12301) 2024-12-05T05:19:13,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/24c808dfa3a34d31bc7f324a9bc6e51e 2024-12-05T05:19:13,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/6f52d699e43b48bfbd5130cd5ccd3326 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6f52d699e43b48bfbd5130cd5ccd3326 2024-12-05T05:19:13,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6f52d699e43b48bfbd5130cd5ccd3326, entries=150, sequenceid=399, filesize=30.5 K 2024-12-05T05:19:13,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/3834bdce9d0a4c399b32608768facdb1 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3834bdce9d0a4c399b32608768facdb1 2024-12-05T05:19:13,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3834bdce9d0a4c399b32608768facdb1, entries=150, sequenceid=399, filesize=12.0 K 2024-12-05T05:19:13,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/24c808dfa3a34d31bc7f324a9bc6e51e as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/24c808dfa3a34d31bc7f324a9bc6e51e 2024-12-05T05:19:13,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/24c808dfa3a34d31bc7f324a9bc6e51e, entries=150, sequenceid=399, filesize=12.0 K 2024-12-05T05:19:13,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for ae3f93f3f6ee0cd97fcedac40066c066 in 1676ms, sequenceid=399, compaction requested=true 2024-12-05T05:19:13,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:13,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:A, priority=-2147483648, current under compaction store size is 1 2024-12-05T05:19:13,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:13,080 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:13,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:B, priority=-2147483648, current under compaction store size is 2 2024-12-05T05:19:13,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:13,080 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:13,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ae3f93f3f6ee0cd97fcedac40066c066:C, priority=-2147483648, current under compaction store size is 3 2024-12-05T05:19:13,080 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:13,081 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:13,081 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:13,081 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/B is initiating minor compaction (all files) 2024-12-05T05:19:13,081 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/A is initiating minor compaction (all files) 2024-12-05T05:19:13,081 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/A in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,081 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/B in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,081 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8ac3db1c7eee43fe945c734770393483, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/32ee0876d9d747d5bfc3e441294440c3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3834bdce9d0a4c399b32608768facdb1] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=36.7 K 2024-12-05T05:19:13,081 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/45c4c315cf424e1da8836edc86bf6d01, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f72fc149db9e4a06b5720ee56570db2b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6f52d699e43b48bfbd5130cd5ccd3326] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=92.2 K 2024-12-05T05:19:13,081 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,081 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. files: [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/45c4c315cf424e1da8836edc86bf6d01, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f72fc149db9e4a06b5720ee56570db2b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6f52d699e43b48bfbd5130cd5ccd3326] 2024-12-05T05:19:13,081 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ac3db1c7eee43fe945c734770393483, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375949032 2024-12-05T05:19:13,082 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45c4c315cf424e1da8836edc86bf6d01, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375949032 2024-12-05T05:19:13,082 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 32ee0876d9d747d5bfc3e441294440c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733375949078 2024-12-05T05:19:13,082 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting f72fc149db9e4a06b5720ee56570db2b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733375949078 2024-12-05T05:19:13,082 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 3834bdce9d0a4c399b32608768facdb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733375950266 2024-12-05T05:19:13,082 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f52d699e43b48bfbd5130cd5ccd3326, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733375950266 2024-12-05T05:19:13,088 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:13,089 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#B#compaction#543 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:13,089 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/67c3c52f09174e2fa4da2cabdd23191a is 50, key is test_row_0/B:col10/1733375950266/Put/seqid=0 2024-12-05T05:19:13,093 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241205c4576fc3aa194c0a8e6f795d999b4447_ae3f93f3f6ee0cd97fcedac40066c066 store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:13,094 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241205c4576fc3aa194c0a8e6f795d999b4447_ae3f93f3f6ee0cd97fcedac40066c066, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:13,094 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205c4576fc3aa194c0a8e6f795d999b4447_ae3f93f3f6ee0cd97fcedac40066c066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:13,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742472_1648 (size=13085) 2024-12-05T05:19:13,113 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/67c3c52f09174e2fa4da2cabdd23191a as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/67c3c52f09174e2fa4da2cabdd23191a 2024-12-05T05:19:13,120 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/B of ae3f93f3f6ee0cd97fcedac40066c066 into 67c3c52f09174e2fa4da2cabdd23191a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:13,120 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:13,120 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/B, priority=13, startTime=1733375953080; duration=0sec 2024-12-05T05:19:13,120 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-05T05:19:13,120 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:B 2024-12-05T05:19:13,120 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-05T05:19:13,121 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-05T05:19:13,121 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1541): ae3f93f3f6ee0cd97fcedac40066c066/C is initiating minor compaction (all files) 2024-12-05T05:19:13,121 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ae3f93f3f6ee0cd97fcedac40066c066/C in TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,121 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3fa5cad36bd34e948662cc982cdad741, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48230c08ce3e420a86fbb20a81d9aee3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/24c808dfa3a34d31bc7f324a9bc6e51e] into tmpdir=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp, totalSize=36.7 K 2024-12-05T05:19:13,123 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fa5cad36bd34e948662cc982cdad741, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1733375949032 2024-12-05T05:19:13,123 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 48230c08ce3e420a86fbb20a81d9aee3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1733375949078 2024-12-05T05:19:13,123 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] compactions.Compactor(225): Compacting 24c808dfa3a34d31bc7f324a9bc6e51e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1733375950266 2024-12-05T05:19:13,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742473_1649 (size=4469) 2024-12-05T05:19:13,129 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#C#compaction#545 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:13,129 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/9e3031c2891b423f966dc1e27e69c9fb is 50, key is test_row_0/C:col10/1733375950266/Put/seqid=0 2024-12-05T05:19:13,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742474_1650 (size=13085) 2024-12-05T05:19:13,145 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/9e3031c2891b423f966dc1e27e69c9fb as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9e3031c2891b423f966dc1e27e69c9fb 2024-12-05T05:19:13,148 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/C of ae3f93f3f6ee0cd97fcedac40066c066 into 9e3031c2891b423f966dc1e27e69c9fb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:13,148 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:13,148 INFO [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/C, priority=13, startTime=1733375953080; duration=0sec 2024-12-05T05:19:13,148 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:13,148 DEBUG [RS:0;8edcaf2ecc84:36233-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:C 2024-12-05T05:19:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-05T05:19:13,173 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:19:13,173 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-12-05T05:19:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-05T05:19:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T05:19:13,174 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-05T05:19:13,175 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-05T05:19:13,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-05T05:19:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T05:19:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(8855): Flush requested on ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:13,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-12-05T05:19:13,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:13,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:13,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:13,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:13,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:13,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-05T05:19:13,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205de2f2a610d3040fbad19804d5eb20e57_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375951471/Put/seqid=0 2024-12-05T05:19:13,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742475_1651 (size=14994) 2024-12-05T05:19:13,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T05:19:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:13,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376013337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:13,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376013443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:13,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:13,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T05:19:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T05:19:13,526 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ae3f93f3f6ee0cd97fcedac40066c066#A#compaction#544 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-05T05:19:13,527 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/87290186dc3b420c8f8a012f22d62906 is 175, key is test_row_0/A:col10/1733375950266/Put/seqid=0 2024-12-05T05:19:13,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742476_1652 (size=32039) 2024-12-05T05:19:13,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53544 deadline: 1733376013623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:13,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53564 deadline: 1733376013623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:13,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:13,625 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53542 deadline: 1733376013623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,625 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:13,625 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:13,626 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_0' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:13,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T05:19:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53536 deadline: 1733376013633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:13,636 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 8edcaf2ecc84,36233,1733375771401 for 'test_row_2' in ae3f93f3f6ee0cd97fcedac40066c066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-05T05:19:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,636 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:13,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376013652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:13,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:13,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241205de2f2a610d3040fbad19804d5eb20e57_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de2f2a610d3040fbad19804d5eb20e57_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:13,733 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/28fdcc84a9cf48968c4a14e48c9a4d88, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:13,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/28fdcc84a9cf48968c4a14e48c9a4d88 is 175, key is test_row_0/A:col10/1733375951471/Put/seqid=0 2024-12-05T05:19:13,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742477_1653 (size=39949) 2024-12-05T05:19:13,775 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=438, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/28fdcc84a9cf48968c4a14e48c9a4d88 2024-12-05T05:19:13,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/e9d5ced367d14dbd83d1c875f1726b00 is 50, key is test_row_0/B:col10/1733375951471/Put/seqid=0 2024-12-05T05:19:13,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T05:19:13,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:13,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742478_1654 (size=12301) 2024-12-05T05:19:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T05:19:13,935 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/87290186dc3b420c8f8a012f22d62906 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/87290186dc3b420c8f8a012f22d62906 2024-12-05T05:19:13,939 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ae3f93f3f6ee0cd97fcedac40066c066/A of ae3f93f3f6ee0cd97fcedac40066c066 into 87290186dc3b420c8f8a012f22d62906(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-05T05:19:13,939 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:13,939 INFO [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., storeName=ae3f93f3f6ee0cd97fcedac40066c066/A, priority=13, startTime=1733375953080; duration=0sec 2024-12-05T05:19:13,939 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-05T05:19:13,939 DEBUG [RS:0;8edcaf2ecc84:36233-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ae3f93f3f6ee0cd97fcedac40066c066:A 2024-12-05T05:19:13,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T05:19:13,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:13,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:13,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:13,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-05T05:19:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36233 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:53508 deadline: 1733376013963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 , the old value is region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-05T05:19:13,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., hostname=8edcaf2ecc84,36233,1733375771401, seqNum=5 because the exception is null or not the one we care about 2024-12-05T05:19:14,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T05:19:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:14,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. as already flushing 2024-12-05T05:19:14,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:14,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:14,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T05:19:14,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/e9d5ced367d14dbd83d1c875f1726b00 2024-12-05T05:19:14,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/c3dd7703694240f88ba3c4877c55b0c6 is 50, key is test_row_0/C:col10/1733375951471/Put/seqid=0 2024-12-05T05:19:14,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742479_1655 (size=12301) 2024-12-05T05:19:14,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/c3dd7703694240f88ba3c4877c55b0c6 2024-12-05T05:19:14,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/28fdcc84a9cf48968c4a14e48c9a4d88 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/28fdcc84a9cf48968c4a14e48c9a4d88 2024-12-05T05:19:14,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/28fdcc84a9cf48968c4a14e48c9a4d88, entries=200, sequenceid=438, filesize=39.0 K 2024-12-05T05:19:14,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/e9d5ced367d14dbd83d1c875f1726b00 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/e9d5ced367d14dbd83d1c875f1726b00 2024-12-05T05:19:14,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/e9d5ced367d14dbd83d1c875f1726b00, entries=150, sequenceid=438, filesize=12.0 K 2024-12-05T05:19:14,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/c3dd7703694240f88ba3c4877c55b0c6 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c3dd7703694240f88ba3c4877c55b0c6 2024-12-05T05:19:14,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,237 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c3dd7703694240f88ba3c4877c55b0c6, entries=150, sequenceid=438, filesize=12.0 K 2024-12-05T05:19:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for ae3f93f3f6ee0cd97fcedac40066c066 in 922ms, sequenceid=438, compaction requested=false 2024-12-05T05:19:14,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,244 INFO [Thread-2498 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2498. 2024-12-05T05:19:14,245 DEBUG [Thread-2498 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:14,245 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,245 DEBUG [Thread-2498 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:14,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,245 INFO [Thread-2492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2492. 2024-12-05T05:19:14,245 DEBUG [Thread-2492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:14,245 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,245 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,245 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,246 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-12-05T05:19:14,246 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:14,246 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,246 INFO [Thread-2490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2490. 2024-12-05T05:19:14,246 DEBUG [Thread-2490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:14,246 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,246 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:14,246 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36233 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-05T05:19:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,248 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T05:19:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:14,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:14,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059131c7766c8a47299d70c85e05762dcf_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375953329/Put/seqid=0 2024-12-05T05:19:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36233 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,256 INFO [Thread-2494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2494. 2024-12-05T05:19:14,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742480_1656 (size=9914) 2024-12-05T05:19:14,256 DEBUG [Thread-2494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:14,256 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,256 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:14,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:14,259 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412059131c7766c8a47299d70c85e05762dcf_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059131c7766c8a47299d70c85e05762dcf_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:14,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/acfe95c180854da282b616ecae2a9924, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:14,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/acfe95c180854da282b616ecae2a9924 is 175, key is test_row_0/A:col10/1733375953329/Put/seqid=0 2024-12-05T05:19:14,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742481_1657 (size=22561) 2024-12-05T05:19:14,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T05:19:14,473 INFO [Thread-2487 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2487. 2024-12-05T05:19:14,473 DEBUG [Thread-2487 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:14,473 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,473 DEBUG [Thread-2487 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:14,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:14,663 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=447, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/acfe95c180854da282b616ecae2a9924 2024-12-05T05:19:14,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/4555843085024865948cb4b3e94c4153 is 50, key is test_row_0/B:col10/1733375953329/Put/seqid=0 2024-12-05T05:19:14,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742482_1658 (size=9857) 2024-12-05T05:19:15,072 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/4555843085024865948cb4b3e94c4153 2024-12-05T05:19:15,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/69df433a67804412aaab3786ce98e58c is 50, key is test_row_0/C:col10/1733375953329/Put/seqid=0 2024-12-05T05:19:15,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742483_1659 (size=9857) 2024-12-05T05:19:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T05:19:15,481 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=447 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/69df433a67804412aaab3786ce98e58c 2024-12-05T05:19:15,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/acfe95c180854da282b616ecae2a9924 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/acfe95c180854da282b616ecae2a9924 2024-12-05T05:19:15,486 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/acfe95c180854da282b616ecae2a9924, entries=100, sequenceid=447, filesize=22.0 K 2024-12-05T05:19:15,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/4555843085024865948cb4b3e94c4153 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/4555843085024865948cb4b3e94c4153 2024-12-05T05:19:15,490 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/4555843085024865948cb4b3e94c4153, entries=100, sequenceid=447, filesize=9.6 K 2024-12-05T05:19:15,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/69df433a67804412aaab3786ce98e58c as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/69df433a67804412aaab3786ce98e58c 2024-12-05T05:19:15,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/69df433a67804412aaab3786ce98e58c, entries=100, sequenceid=447, filesize=9.6 K 2024-12-05T05:19:15,493 INFO [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=6.71 KB/6870 for ae3f93f3f6ee0cd97fcedac40066c066 in 1245ms, sequenceid=447, compaction requested=true 2024-12-05T05:19:15,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for ae3f93f3f6ee0cd97fcedac40066c066: 2024-12-05T05:19:15,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:15,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/8edcaf2ecc84:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-05T05:19:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-05T05:19:15,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-05T05:19:15,496 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-12-05T05:19:15,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.3230 sec 2024-12-05T05:19:15,634 INFO [Thread-2483 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2483. 2024-12-05T05:19:15,634 DEBUG [Thread-2483 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:15,634 INFO [Thread-2481 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2481. 2024-12-05T05:19:15,634 DEBUG [Thread-2483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:15,634 DEBUG [Thread-2481 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:15,634 DEBUG [Thread-2481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:15,634 DEBUG [Thread-2483 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:15,634 DEBUG [Thread-2481 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:15,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:15,634 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:15,654 INFO [Thread-2485 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2485. 2024-12-05T05:19:15,654 DEBUG [Thread-2485 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:15,654 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:15,654 DEBUG [Thread-2485 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:15,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:15,664 INFO [Thread-2479 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2479. 2024-12-05T05:19:15,664 DEBUG [Thread-2479 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-05T05:19:15,664 DEBUG [Thread-2479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:15,664 DEBUG [Thread-2479 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:15,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-05T05:19:17,333 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3665 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3427 2024-12-05T05:19:17,333 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3606 2024-12-05T05:19:17,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3684 2024-12-05T05:19:17,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3407 2024-12-05T05:19:17,334 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-05T05:19:17,334 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T05:19:17,334 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:19:17,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:17,334 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:17,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:17,334 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-12-05T05:19:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-05T05:19:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T05:19:17,337 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375957337"}]},"ts":"1733375957337"} 2024-12-05T05:19:17,338 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-05T05:19:17,339 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-05T05:19:17,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-05T05:19:17,340 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, UNASSIGN}] 2024-12-05T05:19:17,341 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, UNASSIGN 2024-12-05T05:19:17,341 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=ae3f93f3f6ee0cd97fcedac40066c066, regionState=CLOSING, regionLocation=8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:17,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, UNASSIGN because future has completed 2024-12-05T05:19:17,342 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-05T05:19:17,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401}] 2024-12-05T05:19:17,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T05:19:17,494 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing ae3f93f3f6ee0cd97fcedac40066c066, disabling compactions & flushes 2024-12-05T05:19:17,494 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. after waiting 0 ms 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:17,494 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing ae3f93f3f6ee0cd97fcedac40066c066 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=A 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=B 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:17,494 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ae3f93f3f6ee0cd97fcedac40066c066, store=C 2024-12-05T05:19:17,495 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-05T05:19:17,499 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054124040fd1f2423bb7dbc87a303dfb58_ae3f93f3f6ee0cd97fcedac40066c066 is 50, key is test_row_0/A:col10/1733375955653/Put/seqid=0 2024-12-05T05:19:17,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742484_1660 (size=12454) 2024-12-05T05:19:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T05:19:17,902 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T05:19:17,905 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412054124040fd1f2423bb7dbc87a303dfb58_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054124040fd1f2423bb7dbc87a303dfb58_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:17,905 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/f9c1d0f65e2f4823a52167d1e4e702bf, store: [table=TestAcidGuarantees family=A region=ae3f93f3f6ee0cd97fcedac40066c066] 2024-12-05T05:19:17,906 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/f9c1d0f65e2f4823a52167d1e4e702bf is 175, key is test_row_0/A:col10/1733375955653/Put/seqid=0 2024-12-05T05:19:17,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742485_1661 (size=31255) 2024-12-05T05:19:17,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T05:19:18,309 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=455, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/f9c1d0f65e2f4823a52167d1e4e702bf 2024-12-05T05:19:18,314 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a88640772d4440419eea9c3b99c02266 is 50, key is test_row_0/B:col10/1733375955653/Put/seqid=0 2024-12-05T05:19:18,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742486_1662 (size=12301) 2024-12-05T05:19:18,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T05:19:18,717 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a88640772d4440419eea9c3b99c02266 2024-12-05T05:19:18,722 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/3455048f28a64ad6a32aa1b42c185a33 is 50, key is test_row_0/C:col10/1733375955653/Put/seqid=0 2024-12-05T05:19:18,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742487_1663 (size=12301) 2024-12-05T05:19:19,125 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/3455048f28a64ad6a32aa1b42c185a33 2024-12-05T05:19:19,129 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/A/f9c1d0f65e2f4823a52167d1e4e702bf as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f9c1d0f65e2f4823a52167d1e4e702bf 2024-12-05T05:19:19,132 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f9c1d0f65e2f4823a52167d1e4e702bf, entries=150, sequenceid=455, filesize=30.5 K 2024-12-05T05:19:19,132 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/B/a88640772d4440419eea9c3b99c02266 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a88640772d4440419eea9c3b99c02266 2024-12-05T05:19:19,134 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a88640772d4440419eea9c3b99c02266, entries=150, sequenceid=455, filesize=12.0 K 2024-12-05T05:19:19,135 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/.tmp/C/3455048f28a64ad6a32aa1b42c185a33 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3455048f28a64ad6a32aa1b42c185a33 2024-12-05T05:19:19,137 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3455048f28a64ad6a32aa1b42c185a33, entries=150, sequenceid=455, filesize=12.0 K 2024-12-05T05:19:19,138 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ae3f93f3f6ee0cd97fcedac40066c066 in 1644ms, sequenceid=455, compaction requested=true 2024-12-05T05:19:19,138 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/01a4cf1ec4d948ee9acfd579f0aaf4da, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/61ec288a7b3e46e1af0327a9bb6b878d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/4d01da6dad794c1ea5202205162d471e, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bf56dd1064c347159d49102047ac9b20, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/e2653b8e38454b4bb30ea6a02e840e81, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/57e78050473643b89dbb05fd125a65d0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/2f26c1d270ee4e2cbd385e3abf3ca860, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/181805823d1645ffb0764e298c51318c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/b48accf1a8d54784bed34577adf90fea, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6b4f0d2ef0b447f2a3196e5d8f7397e7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/96f423e911fb453380bdbc4cae82b909, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/0b52a32353ac4e2faea536ae975275db, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/41302f8225344511b954c8ba6ce8f2a3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bb8e59a713364c54b4c8fb95f567c4a7, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/fea71c23797547d9bccfef5c7b450029, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/36a4b5abfa794ed0955ef8376e6dea46, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/642f79148738494786d87f2bca6ecf8a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/d5cd230573ee453285eedfa3ca7b995a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/45c4c315cf424e1da8836edc86bf6d01, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/1279a10179a44003bf43e4021682b99d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f72fc149db9e4a06b5720ee56570db2b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6f52d699e43b48bfbd5130cd5ccd3326] to archive 2024-12-05T05:19:19,139 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:19:19,140 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/01a4cf1ec4d948ee9acfd579f0aaf4da to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/01a4cf1ec4d948ee9acfd579f0aaf4da 2024-12-05T05:19:19,141 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/61ec288a7b3e46e1af0327a9bb6b878d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/61ec288a7b3e46e1af0327a9bb6b878d 2024-12-05T05:19:19,142 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/4d01da6dad794c1ea5202205162d471e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/4d01da6dad794c1ea5202205162d471e 2024-12-05T05:19:19,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bf56dd1064c347159d49102047ac9b20 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bf56dd1064c347159d49102047ac9b20 2024-12-05T05:19:19,143 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/e2653b8e38454b4bb30ea6a02e840e81 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/e2653b8e38454b4bb30ea6a02e840e81 2024-12-05T05:19:19,144 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/57e78050473643b89dbb05fd125a65d0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/57e78050473643b89dbb05fd125a65d0 2024-12-05T05:19:19,145 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/2f26c1d270ee4e2cbd385e3abf3ca860 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/2f26c1d270ee4e2cbd385e3abf3ca860 2024-12-05T05:19:19,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/181805823d1645ffb0764e298c51318c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/181805823d1645ffb0764e298c51318c 2024-12-05T05:19:19,146 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/b48accf1a8d54784bed34577adf90fea to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/b48accf1a8d54784bed34577adf90fea 2024-12-05T05:19:19,147 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6b4f0d2ef0b447f2a3196e5d8f7397e7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6b4f0d2ef0b447f2a3196e5d8f7397e7 2024-12-05T05:19:19,148 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/96f423e911fb453380bdbc4cae82b909 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/96f423e911fb453380bdbc4cae82b909 2024-12-05T05:19:19,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/0b52a32353ac4e2faea536ae975275db to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/0b52a32353ac4e2faea536ae975275db 2024-12-05T05:19:19,149 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/41302f8225344511b954c8ba6ce8f2a3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/41302f8225344511b954c8ba6ce8f2a3 2024-12-05T05:19:19,150 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bb8e59a713364c54b4c8fb95f567c4a7 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/bb8e59a713364c54b4c8fb95f567c4a7 2024-12-05T05:19:19,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/fea71c23797547d9bccfef5c7b450029 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/fea71c23797547d9bccfef5c7b450029 2024-12-05T05:19:19,151 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/36a4b5abfa794ed0955ef8376e6dea46 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/36a4b5abfa794ed0955ef8376e6dea46 2024-12-05T05:19:19,152 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/dd1a0fdf3c6e4dbf88d208bf625fa7d9 2024-12-05T05:19:19,153 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/642f79148738494786d87f2bca6ecf8a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/642f79148738494786d87f2bca6ecf8a 2024-12-05T05:19:19,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/d5cd230573ee453285eedfa3ca7b995a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/d5cd230573ee453285eedfa3ca7b995a 2024-12-05T05:19:19,154 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/45c4c315cf424e1da8836edc86bf6d01 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/45c4c315cf424e1da8836edc86bf6d01 2024-12-05T05:19:19,155 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/1279a10179a44003bf43e4021682b99d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/1279a10179a44003bf43e4021682b99d 2024-12-05T05:19:19,156 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f72fc149db9e4a06b5720ee56570db2b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f72fc149db9e4a06b5720ee56570db2b 2024-12-05T05:19:19,157 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6f52d699e43b48bfbd5130cd5ccd3326 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/6f52d699e43b48bfbd5130cd5ccd3326 2024-12-05T05:19:19,158 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b3d76038436746b6a66f31da725308ff, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ed9fd7b2d97e4aec8451b9db75c3403d, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/c973069fde17468daeba0470db43df47, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8f5588f919684494ba97158178858292, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/5b120eb97cd64c76b093016d8e445157, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b9e0dd70013743e083e7c60ff3956678, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8da81a85ec9e4698973fdafa97550c54, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/272068a567d04cb2969d8c510e414a60, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/f458acee1da74397b29a34223ca65628, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/6c41c802e0a345788bd2b59ba58ffb7c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3fed77b92e144d5b9f3fd97c7b9c1b48, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/0f8c10b1385c4e94a1167be2f28861ee, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ad6ce8945a3b45bea1ab6d87e5974066, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/73c9c97f90334ff6924461c5e02785a2, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/d510519c54ac4261a4f41e5f3fda93b4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ffab145383554835881c6ad81a1815a1, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a4fe5ffca13c497581f692da881fa710, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/2d2a4cd8c6714f7b96af5a710c7cdc1f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8ac3db1c7eee43fe945c734770393483, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a9777f45c59e4bb5aa5e1c80784d2a29, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/32ee0876d9d747d5bfc3e441294440c3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3834bdce9d0a4c399b32608768facdb1] to archive 2024-12-05T05:19:19,159 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:19:19,160 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b3d76038436746b6a66f31da725308ff to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b3d76038436746b6a66f31da725308ff 2024-12-05T05:19:19,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/7a63c136fd3c4c2c88b1ca7a2d1c0e5c 2024-12-05T05:19:19,161 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ed9fd7b2d97e4aec8451b9db75c3403d to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ed9fd7b2d97e4aec8451b9db75c3403d 2024-12-05T05:19:19,162 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/c973069fde17468daeba0470db43df47 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/c973069fde17468daeba0470db43df47 2024-12-05T05:19:19,163 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8f5588f919684494ba97158178858292 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8f5588f919684494ba97158178858292 2024-12-05T05:19:19,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/5b120eb97cd64c76b093016d8e445157 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/5b120eb97cd64c76b093016d8e445157 2024-12-05T05:19:19,164 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b9e0dd70013743e083e7c60ff3956678 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/b9e0dd70013743e083e7c60ff3956678 2024-12-05T05:19:19,165 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8da81a85ec9e4698973fdafa97550c54 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8da81a85ec9e4698973fdafa97550c54 2024-12-05T05:19:19,166 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/272068a567d04cb2969d8c510e414a60 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/272068a567d04cb2969d8c510e414a60 2024-12-05T05:19:19,167 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/f458acee1da74397b29a34223ca65628 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/f458acee1da74397b29a34223ca65628 2024-12-05T05:19:19,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/6c41c802e0a345788bd2b59ba58ffb7c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/6c41c802e0a345788bd2b59ba58ffb7c 2024-12-05T05:19:19,168 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3fed77b92e144d5b9f3fd97c7b9c1b48 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3fed77b92e144d5b9f3fd97c7b9c1b48 2024-12-05T05:19:19,169 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/0f8c10b1385c4e94a1167be2f28861ee to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/0f8c10b1385c4e94a1167be2f28861ee 2024-12-05T05:19:19,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ad6ce8945a3b45bea1ab6d87e5974066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ad6ce8945a3b45bea1ab6d87e5974066 2024-12-05T05:19:19,170 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/73c9c97f90334ff6924461c5e02785a2 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/73c9c97f90334ff6924461c5e02785a2 2024-12-05T05:19:19,171 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/d510519c54ac4261a4f41e5f3fda93b4 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/d510519c54ac4261a4f41e5f3fda93b4 2024-12-05T05:19:19,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ffab145383554835881c6ad81a1815a1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/ffab145383554835881c6ad81a1815a1 2024-12-05T05:19:19,172 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a4fe5ffca13c497581f692da881fa710 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a4fe5ffca13c497581f692da881fa710 2024-12-05T05:19:19,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/2d2a4cd8c6714f7b96af5a710c7cdc1f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/2d2a4cd8c6714f7b96af5a710c7cdc1f 2024-12-05T05:19:19,173 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8ac3db1c7eee43fe945c734770393483 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/8ac3db1c7eee43fe945c734770393483 2024-12-05T05:19:19,174 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a9777f45c59e4bb5aa5e1c80784d2a29 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a9777f45c59e4bb5aa5e1c80784d2a29 2024-12-05T05:19:19,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/32ee0876d9d747d5bfc3e441294440c3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/32ee0876d9d747d5bfc3e441294440c3 2024-12-05T05:19:19,175 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3834bdce9d0a4c399b32608768facdb1 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/3834bdce9d0a4c399b32608768facdb1 2024-12-05T05:19:19,176 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4449a498234e4cc88dad014fcbbf86f5, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e3201bcb1e2742cea2afbaa606ea2c23, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0f03aac9b2d54526b16eb154ba75c095, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6672698ca8014b7aa08f1af280c28835, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/53abe7698123446f91761336981ad60b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4a3262617164440cae0ddfe78649d23c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/a7c8bd4aa6364ec4b6f164c03bd9ef67, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/be66c9ef1717429cb58c6276b83d527f, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/ac4460b9540e457eb64ceed27a859bd3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/f62dab9561f0490b9432b97dda29c18b, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48176596775c43b1895c19de17a678fa, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e2c86f75efe947c997e73a526e34d820, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e68061b88e0443c195f8b542a0fe25e0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/552d0a7f86a14f969893d8b993ea22b0, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/89e7233325c7412782fef5a4cff3d4d4, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0182af853e544e5499b4ba968bc1e0a9, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6a02a20c0b40454a8a714aede4713f2c, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9519834a24fa43ae8639b87b3c7ae180, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/1a02e61a551b4083a662bcb423d3b564, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3fa5cad36bd34e948662cc982cdad741, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c818b53cb2e443dd89f5eb01411b0b6a, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48230c08ce3e420a86fbb20a81d9aee3, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/24c808dfa3a34d31bc7f324a9bc6e51e] to archive 2024-12-05T05:19:19,177 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-05T05:19:19,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4449a498234e4cc88dad014fcbbf86f5 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4449a498234e4cc88dad014fcbbf86f5 2024-12-05T05:19:19,178 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e3201bcb1e2742cea2afbaa606ea2c23 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e3201bcb1e2742cea2afbaa606ea2c23 2024-12-05T05:19:19,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0f03aac9b2d54526b16eb154ba75c095 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0f03aac9b2d54526b16eb154ba75c095 2024-12-05T05:19:19,179 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6672698ca8014b7aa08f1af280c28835 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6672698ca8014b7aa08f1af280c28835 2024-12-05T05:19:19,180 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/53abe7698123446f91761336981ad60b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/53abe7698123446f91761336981ad60b 2024-12-05T05:19:19,181 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4a3262617164440cae0ddfe78649d23c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/4a3262617164440cae0ddfe78649d23c 2024-12-05T05:19:19,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/a7c8bd4aa6364ec4b6f164c03bd9ef67 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/a7c8bd4aa6364ec4b6f164c03bd9ef67 2024-12-05T05:19:19,182 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/be66c9ef1717429cb58c6276b83d527f to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/be66c9ef1717429cb58c6276b83d527f 2024-12-05T05:19:19,183 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/ac4460b9540e457eb64ceed27a859bd3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/ac4460b9540e457eb64ceed27a859bd3 2024-12-05T05:19:19,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/f62dab9561f0490b9432b97dda29c18b to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/f62dab9561f0490b9432b97dda29c18b 2024-12-05T05:19:19,184 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48176596775c43b1895c19de17a678fa to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48176596775c43b1895c19de17a678fa 2024-12-05T05:19:19,185 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e2c86f75efe947c997e73a526e34d820 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e2c86f75efe947c997e73a526e34d820 2024-12-05T05:19:19,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e68061b88e0443c195f8b542a0fe25e0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/e68061b88e0443c195f8b542a0fe25e0 2024-12-05T05:19:19,186 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/552d0a7f86a14f969893d8b993ea22b0 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/552d0a7f86a14f969893d8b993ea22b0 2024-12-05T05:19:19,187 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/89e7233325c7412782fef5a4cff3d4d4 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/89e7233325c7412782fef5a4cff3d4d4 2024-12-05T05:19:19,188 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0182af853e544e5499b4ba968bc1e0a9 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/0182af853e544e5499b4ba968bc1e0a9 2024-12-05T05:19:19,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6a02a20c0b40454a8a714aede4713f2c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/6a02a20c0b40454a8a714aede4713f2c 2024-12-05T05:19:19,189 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9519834a24fa43ae8639b87b3c7ae180 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9519834a24fa43ae8639b87b3c7ae180 2024-12-05T05:19:19,190 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/1a02e61a551b4083a662bcb423d3b564 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/1a02e61a551b4083a662bcb423d3b564 2024-12-05T05:19:19,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3fa5cad36bd34e948662cc982cdad741 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3fa5cad36bd34e948662cc982cdad741 2024-12-05T05:19:19,191 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c818b53cb2e443dd89f5eb01411b0b6a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c818b53cb2e443dd89f5eb01411b0b6a 2024-12-05T05:19:19,192 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48230c08ce3e420a86fbb20a81d9aee3 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/48230c08ce3e420a86fbb20a81d9aee3 2024-12-05T05:19:19,193 DEBUG [StoreCloser-TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/24c808dfa3a34d31bc7f324a9bc6e51e to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/24c808dfa3a34d31bc7f324a9bc6e51e 2024-12-05T05:19:19,196 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/recovered.edits/458.seqid, newMaxSeqId=458, maxSeqId=4 2024-12-05T05:19:19,196 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. 2024-12-05T05:19:19,196 DEBUG [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for ae3f93f3f6ee0cd97fcedac40066c066: Waiting for close lock at 1733375957494Running coprocessor pre-close hooks at 1733375957494Disabling compacts and flushes for region at 1733375957494Disabling writes for close at 1733375957494Obtaining lock to block concurrent updates at 1733375957494Preparing flush snapshotting stores in ae3f93f3f6ee0cd97fcedac40066c066 at 1733375957494Finished memstore snapshotting TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733375957495 (+1 ms)Flushing stores of TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066. at 1733375957495Flushing ae3f93f3f6ee0cd97fcedac40066c066/A: creating writer at 1733375957495Flushing mob file ae3f93f3f6ee0cd97fcedac40066c066/A: appending metadata at 1733375957498 (+3 ms)Flushing mob file ae3f93f3f6ee0cd97fcedac40066c066/A: closing flushed file at 1733375957498Flushing ae3f93f3f6ee0cd97fcedac40066c066/A: appending metadata at 1733375957905 (+407 ms)Flushing ae3f93f3f6ee0cd97fcedac40066c066/A: closing flushed file at 1733375957906 (+1 ms)Flushing ae3f93f3f6ee0cd97fcedac40066c066/B: creating writer at 1733375958312 (+406 ms)Flushing ae3f93f3f6ee0cd97fcedac40066c066/B: appending metadata at 1733375958314 (+2 ms)Flushing ae3f93f3f6ee0cd97fcedac40066c066/B: closing flushed file at 1733375958314Flushing ae3f93f3f6ee0cd97fcedac40066c066/C: creating writer at 1733375958720 (+406 ms)Flushing ae3f93f3f6ee0cd97fcedac40066c066/C: appending metadata at 1733375958722 (+2 ms)Flushing ae3f93f3f6ee0cd97fcedac40066c066/C: closing flushed file at 1733375958722Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f08c5bf: reopening flushed file at 1733375959128 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15940d2f: reopening flushed file at 1733375959132 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d5b5865: reopening flushed file at 1733375959134 (+2 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ae3f93f3f6ee0cd97fcedac40066c066 in 1644ms, sequenceid=455, compaction requested=true at 1733375959138 (+4 ms)Writing region close event to WAL at 1733375959194 (+56 ms)Running coprocessor post-close hooks at 1733375959196 (+2 ms)Closed at 1733375959196 2024-12-05T05:19:19,197 INFO [RS_CLOSE_REGION-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,198 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=ae3f93f3f6ee0cd97fcedac40066c066, regionState=CLOSED 2024-12-05T05:19:19,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 because future has completed 2024-12-05T05:19:19,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-05T05:19:19,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure ae3f93f3f6ee0cd97fcedac40066c066, server=8edcaf2ecc84,36233,1733375771401 in 1.8570 sec 2024-12-05T05:19:19,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-12-05T05:19:19,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ae3f93f3f6ee0cd97fcedac40066c066, UNASSIGN in 1.8600 sec 2024-12-05T05:19:19,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-05T05:19:19,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8630 sec 2024-12-05T05:19:19,204 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733375959204"}]},"ts":"1733375959204"} 2024-12-05T05:19:19,205 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-05T05:19:19,205 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-05T05:19:19,206 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8700 sec 2024-12-05T05:19:19,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-05T05:19:19,483 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:19:19,483 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-12-05T05:19:19,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:19:19,485 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:19:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-05T05:19:19,486 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:19:19,487 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,489 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C, FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/recovered.edits] 2024-12-05T05:19:19,490 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/28fdcc84a9cf48968c4a14e48c9a4d88 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/28fdcc84a9cf48968c4a14e48c9a4d88 2024-12-05T05:19:19,492 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/87290186dc3b420c8f8a012f22d62906 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/87290186dc3b420c8f8a012f22d62906 2024-12-05T05:19:19,492 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/acfe95c180854da282b616ecae2a9924 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/acfe95c180854da282b616ecae2a9924 2024-12-05T05:19:19,493 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f9c1d0f65e2f4823a52167d1e4e702bf to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/A/f9c1d0f65e2f4823a52167d1e4e702bf 2024-12-05T05:19:19,495 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/4555843085024865948cb4b3e94c4153 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/4555843085024865948cb4b3e94c4153 2024-12-05T05:19:19,496 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/67c3c52f09174e2fa4da2cabdd23191a to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/67c3c52f09174e2fa4da2cabdd23191a 2024-12-05T05:19:19,497 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a88640772d4440419eea9c3b99c02266 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/a88640772d4440419eea9c3b99c02266 2024-12-05T05:19:19,498 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/e9d5ced367d14dbd83d1c875f1726b00 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/B/e9d5ced367d14dbd83d1c875f1726b00 2024-12-05T05:19:19,499 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3455048f28a64ad6a32aa1b42c185a33 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/3455048f28a64ad6a32aa1b42c185a33 2024-12-05T05:19:19,499 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/69df433a67804412aaab3786ce98e58c to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/69df433a67804412aaab3786ce98e58c 2024-12-05T05:19:19,500 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9e3031c2891b423f966dc1e27e69c9fb to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/9e3031c2891b423f966dc1e27e69c9fb 2024-12-05T05:19:19,501 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c3dd7703694240f88ba3c4877c55b0c6 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/C/c3dd7703694240f88ba3c4877c55b0c6 2024-12-05T05:19:19,502 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/recovered.edits/458.seqid to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066/recovered.edits/458.seqid 2024-12-05T05:19:19,502 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/default/TestAcidGuarantees/ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,502 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-05T05:19:19,502 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T05:19:19,503 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-05T05:19:19,504 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509752579ef6b46989827fe226b71eeb0_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509752579ef6b46989827fe226b71eeb0_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,505 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509e8d22ad4494b3eb79d660d5c9f42f1_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120509e8d22ad4494b3eb79d660d5c9f42f1_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,506 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050b4ed95c5a2e4781864dbc6be4a34556_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412050b4ed95c5a2e4781864dbc6be4a34556_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,506 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053ffa3c93fa48403b8158ecc6dd6df4d0_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412053ffa3c93fa48403b8158ecc6dd6df4d0_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,507 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054124040fd1f2423bb7dbc87a303dfb58_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412054124040fd1f2423bb7dbc87a303dfb58_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,507 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ff2f9a08ca24d109a390771f464a918_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412055ff2f9a08ca24d109a390771f464a918_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,508 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120572b9f871ed9c4603ac5671acab26f829_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120572b9f871ed9c4603ac5671acab26f829_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,509 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205827958018eaa4046aa7664c699304908_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205827958018eaa4046aa7664c699304908_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,509 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120583f426f082e14441ad5b1f996bdc9e5c_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120583f426f082e14441ad5b1f996bdc9e5c_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,510 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059131c7766c8a47299d70c85e05762dcf_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412059131c7766c8a47299d70c85e05762dcf_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,510 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205977d3ee77d2245c4bacbc72ae5545d6d_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205977d3ee77d2245c4bacbc72ae5545d6d_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,511 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a5f8e16497404835820ed1cef7c85de3_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205a5f8e16497404835820ed1cef7c85de3_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,512 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bbd49cd641844e63aa2bed6f611261fa_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205bbd49cd641844e63aa2bed6f611261fa_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,512 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c140e858400b48eb99f783f50d8e72a2_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205c140e858400b48eb99f783f50d8e72a2_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,513 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de2f2a610d3040fbad19804d5eb20e57_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205de2f2a610d3040fbad19804d5eb20e57_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,513 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205df7af7588dd2463eb29fe7753f4cc02b_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205df7af7588dd2463eb29fe7753f4cc02b_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,514 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e448283bf0c945ce90f290ab69f68174_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205e448283bf0c945ce90f290ab69f68174_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,514 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eb0f6a3557f543da9f18caa0991b17af_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205eb0f6a3557f543da9f18caa0991b17af_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,515 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f0d5e3a19f5d4006aab2f6d60aafb30c_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205f0d5e3a19f5d4006aab2f6d60aafb30c_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,515 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fa161fb27afb4147992b7236553469c6_ae3f93f3f6ee0cd97fcedac40066c066 to hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241205fa161fb27afb4147992b7236553469c6_ae3f93f3f6ee0cd97fcedac40066c066 2024-12-05T05:19:19,516 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-05T05:19:19,517 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:19:19,519 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-05T05:19:19,520 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-05T05:19:19,521 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:19:19,521 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-05T05:19:19,521 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733375959521"}]},"ts":"9223372036854775807"} 2024-12-05T05:19:19,522 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-05T05:19:19,522 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ae3f93f3f6ee0cd97fcedac40066c066, NAME => 'TestAcidGuarantees,,1733375931154.ae3f93f3f6ee0cd97fcedac40066c066.', STARTKEY => '', ENDKEY => ''}] 2024-12-05T05:19:19,522 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-05T05:19:19,522 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733375959522"}]},"ts":"9223372036854775807"} 2024-12-05T05:19:19,523 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-05T05:19:19,524 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-05T05:19:19,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-12-05T05:19:19,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46695 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-05T05:19:19,593 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-05T05:19:19,593 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-05T05:19:19,600 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 246) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=619 (was 589) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2327 (was 2722) 2024-12-05T05:19:19,600 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-05T05:19:19,600 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T05:19:19,601 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:19:19,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:19,601 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:19,601 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-05T05:19:19,601 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T05:19:19,601 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=350595032, stopped=false 2024-12-05T05:19:19,601 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8edcaf2ecc84,46695,1733375770505 2024-12-05T05:19:19,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T05:19:19,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:19:19,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T05:19:19,612 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T05:19:19,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:19:19,613 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T05:19:19,613 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-05T05:19:19,613 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:19:19,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:19,613 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T05:19:19,614 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8edcaf2ecc84,36233,1733375771401' ***** 2024-12-05T05:19:19,614 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-05T05:19:19,614 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T05:19:19,614 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-05T05:19:19,614 INFO [RS:0;8edcaf2ecc84:36233 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T05:19:19,614 INFO [RS:0;8edcaf2ecc84:36233 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T05:19:19,614 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(959): stopping server 8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:19,614 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T05:19:19,615 INFO [RS:0;8edcaf2ecc84:36233 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8edcaf2ecc84:36233. 2024-12-05T05:19:19,615 DEBUG [RS:0;8edcaf2ecc84:36233 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T05:19:19,615 DEBUG [RS:0;8edcaf2ecc84:36233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:19,615 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T05:19:19,615 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T05:19:19,615 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T05:19:19,615 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-05T05:19:19,616 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-05T05:19:19,616 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-05T05:19:19,616 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-05T05:19:19,616 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-05T05:19:19,616 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-05T05:19:19,616 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T05:19:19,616 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T05:19:19,616 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:19,616 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-05T05:19:19,635 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/info/db5fe801938d43e3b8b4b6377be29d43 is 95, key is TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534./info:/1733375801455/DeleteFamily/seqid=0 2024-12-05T05:19:19,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742488_1664 (size=6514) 2024-12-05T05:19:19,640 INFO [regionserver/8edcaf2ecc84:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T05:19:19,816 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:20,016 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:20,038 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/info/db5fe801938d43e3b8b4b6377be29d43 2024-12-05T05:19:20,055 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/ns/42fb627475c94c10b578911247f1cee7 is 93, key is TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534./ns:/1733375801387/DeleteFamily/seqid=0 2024-12-05T05:19:20,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742489_1665 (size=5906) 2024-12-05T05:19:20,163 INFO [regionserver/8edcaf2ecc84:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-05T05:19:20,163 INFO [regionserver/8edcaf2ecc84:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-05T05:19:20,217 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:20,417 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:20,458 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/ns/42fb627475c94c10b578911247f1cee7 2024-12-05T05:19:20,472 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/rep_barrier/f12e44cb89df48519784362035f1d1de is 102, key is TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534./rep_barrier:/1733375801387/DeleteFamily/seqid=0 2024-12-05T05:19:20,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742490_1666 (size=6025) 2024-12-05T05:19:20,617 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:20,817 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:20,876 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/rep_barrier/f12e44cb89df48519784362035f1d1de 2024-12-05T05:19:20,890 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/table/41a903489e3f4b82811926863ace0555 is 96, key is TestAcidGuarantees,,1733375775827.6ed0ce7f5160651ec0c7956ddfe8e534./table:/1733375801387/DeleteFamily/seqid=0 2024-12-05T05:19:20,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742491_1667 (size=5936) 2024-12-05T05:19:20,933 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-05T05:19:21,017 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:21,218 DEBUG [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-05T05:19:21,293 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/table/41a903489e3f4b82811926863ace0555 2024-12-05T05:19:21,296 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/info/db5fe801938d43e3b8b4b6377be29d43 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/info/db5fe801938d43e3b8b4b6377be29d43 2024-12-05T05:19:21,299 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/info/db5fe801938d43e3b8b4b6377be29d43, entries=12, sequenceid=90, filesize=6.4 K 2024-12-05T05:19:21,299 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/ns/42fb627475c94c10b578911247f1cee7 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/ns/42fb627475c94c10b578911247f1cee7 2024-12-05T05:19:21,302 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/ns/42fb627475c94c10b578911247f1cee7, entries=8, sequenceid=90, filesize=5.8 K 2024-12-05T05:19:21,302 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/rep_barrier/f12e44cb89df48519784362035f1d1de as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/rep_barrier/f12e44cb89df48519784362035f1d1de 2024-12-05T05:19:21,305 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/rep_barrier/f12e44cb89df48519784362035f1d1de, entries=6, sequenceid=90, filesize=5.9 K 2024-12-05T05:19:21,305 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/.tmp/table/41a903489e3f4b82811926863ace0555 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/table/41a903489e3f4b82811926863ace0555 2024-12-05T05:19:21,307 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/table/41a903489e3f4b82811926863ace0555, entries=7, sequenceid=90, filesize=5.8 K 2024-12-05T05:19:21,308 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1692ms, sequenceid=90, compaction requested=false 2024-12-05T05:19:21,311 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-05T05:19:21,311 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T05:19:21,312 INFO [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-05T05:19:21,312 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733375959616Running coprocessor pre-close hooks at 1733375959616Disabling compacts and flushes for region at 1733375959616Disabling writes for close at 1733375959616Obtaining lock to block concurrent updates at 1733375959616Preparing flush snapshotting stores in 1588230740 at 1733375959616Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733375959617 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733375959617Flushing 1588230740/info: creating writer at 1733375959617Flushing 1588230740/info: appending metadata at 1733375959634 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733375959634Flushing 1588230740/ns: creating writer at 1733375960041 (+407 ms)Flushing 1588230740/ns: appending metadata at 1733375960054 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733375960054Flushing 1588230740/rep_barrier: creating writer at 1733375960461 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733375960472 (+11 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733375960472Flushing 1588230740/table: creating writer at 1733375960878 (+406 ms)Flushing 1588230740/table: appending metadata at 1733375960890 (+12 ms)Flushing 1588230740/table: closing flushed file at 1733375960890Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a58a1ca: reopening flushed file at 1733375961296 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6104cbc4: reopening flushed file at 1733375961299 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29650135: reopening flushed file at 1733375961302 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e510961: reopening flushed file at 1733375961305 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1692ms, sequenceid=90, compaction requested=false at 1733375961308 (+3 ms)Writing region close event to WAL at 1733375961309 (+1 ms)Running coprocessor post-close hooks at 1733375961311 (+2 ms)Closed at 1733375961312 (+1 ms) 2024-12-05T05:19:21,312 DEBUG [RS_CLOSE_META-regionserver/8edcaf2ecc84:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T05:19:21,418 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(976): stopping server 8edcaf2ecc84,36233,1733375771401; all regions closed. 2024-12-05T05:19:21,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741834_1010 (size=25344) 2024-12-05T05:19:21,424 DEBUG [RS:0;8edcaf2ecc84:36233 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/oldWALs 2024-12-05T05:19:21,424 INFO [RS:0;8edcaf2ecc84:36233 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8edcaf2ecc84%2C36233%2C1733375771401.meta:.meta(num 1733375775293) 2024-12-05T05:19:21,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741833_1009 (size=16236380) 2024-12-05T05:19:21,427 DEBUG [RS:0;8edcaf2ecc84:36233 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/oldWALs 2024-12-05T05:19:21,427 INFO [RS:0;8edcaf2ecc84:36233 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8edcaf2ecc84%2C36233%2C1733375771401:(num 1733375774339) 2024-12-05T05:19:21,427 DEBUG [RS:0;8edcaf2ecc84:36233 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T05:19:21,427 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T05:19:21,427 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T05:19:21,428 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.ChoreService(370): Chore service for: regionserver/8edcaf2ecc84:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-05T05:19:21,428 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T05:19:21,428 INFO [regionserver/8edcaf2ecc84:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T05:19:21,429 INFO [RS:0;8edcaf2ecc84:36233 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36233 2024-12-05T05:19:21,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8edcaf2ecc84,36233,1733375771401 2024-12-05T05:19:21,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T05:19:21,486 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T05:19:21,487 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8edcaf2ecc84,36233,1733375771401] 2024-12-05T05:19:21,497 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8edcaf2ecc84,36233,1733375771401 already deleted, retry=false 2024-12-05T05:19:21,497 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8edcaf2ecc84,36233,1733375771401 expired; onlineServers=0 2024-12-05T05:19:21,497 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8edcaf2ecc84,46695,1733375770505' ***** 2024-12-05T05:19:21,497 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T05:19:21,497 INFO [M:0;8edcaf2ecc84:46695 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-05T05:19:21,497 INFO [M:0;8edcaf2ecc84:46695 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-05T05:19:21,497 DEBUG [M:0;8edcaf2ecc84:46695 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T05:19:21,498 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T05:19:21,498 DEBUG [M:0;8edcaf2ecc84:46695 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T05:19:21,498 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster-HFileCleaner.small.0-1733375773907 {}] cleaner.HFileCleaner(306): Exit Thread[master/8edcaf2ecc84:0:becomeActiveMaster-HFileCleaner.small.0-1733375773907,5,FailOnTimeoutGroup] 2024-12-05T05:19:21,498 DEBUG [master/8edcaf2ecc84:0:becomeActiveMaster-HFileCleaner.large.0-1733375773904 {}] cleaner.HFileCleaner(306): Exit Thread[master/8edcaf2ecc84:0:becomeActiveMaster-HFileCleaner.large.0-1733375773904,5,FailOnTimeoutGroup] 2024-12-05T05:19:21,498 INFO [M:0;8edcaf2ecc84:46695 {}] hbase.ChoreService(370): Chore service for: master/8edcaf2ecc84:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-05T05:19:21,498 INFO [M:0;8edcaf2ecc84:46695 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-05T05:19:21,498 DEBUG [M:0;8edcaf2ecc84:46695 {}] master.HMaster(1795): Stopping service threads 2024-12-05T05:19:21,498 INFO [M:0;8edcaf2ecc84:46695 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T05:19:21,498 INFO [M:0;8edcaf2ecc84:46695 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-05T05:19:21,498 ERROR [M:0;8edcaf2ecc84:46695 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-05T05:19:21,499 INFO [M:0;8edcaf2ecc84:46695 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T05:19:21,499 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T05:19:21,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T05:19:21,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T05:19:21,507 DEBUG [M:0;8edcaf2ecc84:46695 {}] zookeeper.ZKUtil(347): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-05T05:19:21,507 WARN [M:0;8edcaf2ecc84:46695 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-05T05:19:21,508 INFO [M:0;8edcaf2ecc84:46695 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/.lastflushedseqids 2024-12-05T05:19:21,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742492_1668 (size=72) 2024-12-05T05:19:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T05:19:21,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36233-0x101a3eb37930001, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T05:19:21,597 INFO [RS:0;8edcaf2ecc84:36233 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T05:19:21,598 INFO [RS:0;8edcaf2ecc84:36233 {}] regionserver.HRegionServer(1031): Exiting; stopping=8edcaf2ecc84,36233,1733375771401; zookeeper connection closed. 2024-12-05T05:19:21,598 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@23074ae1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@23074ae1 2024-12-05T05:19:21,598 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T05:19:21,916 INFO [M:0;8edcaf2ecc84:46695 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-05T05:19:21,916 INFO [M:0;8edcaf2ecc84:46695 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T05:19:21,916 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T05:19:21,916 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T05:19:21,917 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T05:19:21,917 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T05:19:21,917 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T05:19:21,917 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=771.78 KB heapSize=945.36 KB 2024-12-05T05:19:21,931 DEBUG [M:0;8edcaf2ecc84:46695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/af004b09919e4c808771b1421ba7e21d is 82, key is hbase:meta,,1/info:regioninfo/1733375775400/Put/seqid=0 2024-12-05T05:19:21,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742493_1669 (size=5672) 2024-12-05T05:19:22,335 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2148 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/af004b09919e4c808771b1421ba7e21d 2024-12-05T05:19:22,352 DEBUG [M:0;8edcaf2ecc84:46695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dad72098782f4a989b3f1a70bb0878dd is 2481, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1733375934214/Put/seqid=0 2024-12-05T05:19:22,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742494_1670 (size=44378) 2024-12-05T05:19:22,356 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=771.22 KB at sequenceid=2148 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dad72098782f4a989b3f1a70bb0878dd 2024-12-05T05:19:22,359 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dad72098782f4a989b3f1a70bb0878dd 2024-12-05T05:19:22,371 DEBUG [M:0;8edcaf2ecc84:46695 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10671341564549e5864fb226d16a1691 is 69, key is 8edcaf2ecc84,36233,1733375771401/rs:state/1733375774071/Put/seqid=0 2024-12-05T05:19:22,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073742495_1671 (size=5156) 2024-12-05T05:19:22,774 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2148 (bloomFilter=true), to=hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10671341564549e5864fb226d16a1691 2024-12-05T05:19:22,778 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/af004b09919e4c808771b1421ba7e21d as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/af004b09919e4c808771b1421ba7e21d 2024-12-05T05:19:22,780 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/af004b09919e4c808771b1421ba7e21d, entries=8, sequenceid=2148, filesize=5.5 K 2024-12-05T05:19:22,781 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/dad72098782f4a989b3f1a70bb0878dd as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dad72098782f4a989b3f1a70bb0878dd 2024-12-05T05:19:22,783 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for dad72098782f4a989b3f1a70bb0878dd 2024-12-05T05:19:22,783 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/dad72098782f4a989b3f1a70bb0878dd, entries=164, sequenceid=2148, filesize=43.3 K 2024-12-05T05:19:22,784 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/10671341564549e5864fb226d16a1691 as hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10671341564549e5864fb226d16a1691 2024-12-05T05:19:22,786 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:38835/user/jenkins/test-data/76f7b5fd-8277-9652-5483-07fe6dc71426/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/10671341564549e5864fb226d16a1691, entries=1, sequenceid=2148, filesize=5.0 K 2024-12-05T05:19:22,787 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(3140): Finished flush of dataSize ~771.78 KB/790298, heapSize ~945.06 KB/967744, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 870ms, sequenceid=2148, compaction requested=false 2024-12-05T05:19:22,790 INFO [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T05:19:22,790 DEBUG [M:0;8edcaf2ecc84:46695 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733375961916Disabling compacts and flushes for region at 1733375961916Disabling writes for close at 1733375961917 (+1 ms)Obtaining lock to block concurrent updates at 1733375961917Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733375961917Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=790298, getHeapSize=967984, getOffHeapSize=0, getCellsCount=2337 at 1733375961917Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733375961917Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733375961917Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733375961931 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733375961931Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733375962338 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733375962352 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733375962352Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733375962359 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733375962371 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733375962371Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ad28a2d: reopening flushed file at 1733375962777 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@432feff7: reopening flushed file at 1733375962780 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79dd3e3: reopening flushed file at 1733375962783 (+3 ms)Finished flush of dataSize ~771.78 KB/790298, heapSize ~945.06 KB/967744, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 870ms, sequenceid=2148, compaction requested=false at 1733375962787 (+4 ms)Writing region close event to WAL at 1733375962790 (+3 ms)Closed at 1733375962790 2024-12-05T05:19:22,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35739 is added to blk_1073741830_1006 (size=931093) 2024-12-05T05:19:22,794 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-05T05:19:22,794 INFO [M:0;8edcaf2ecc84:46695 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-05T05:19:22,794 INFO [M:0;8edcaf2ecc84:46695 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46695 2024-12-05T05:19:22,794 INFO [M:0;8edcaf2ecc84:46695 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-05T05:19:23,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T05:19:23,007 INFO [M:0;8edcaf2ecc84:46695 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-05T05:19:23,007 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46695-0x101a3eb37930000, quorum=127.0.0.1:57337, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T05:19:23,043 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@70061d7e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T05:19:23,045 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@70195653{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T05:19:23,045 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T05:19:23,045 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3ad6a89f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T05:19:23,045 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@15b00a0c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/hadoop.log.dir/,STOPPED} 2024-12-05T05:19:23,048 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T05:19:23,048 WARN [BP-896134436-172.17.0.3-1733375766197 heartbeating to localhost/127.0.0.1:38835 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T05:19:23,048 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T05:19:23,048 WARN [BP-896134436-172.17.0.3-1733375766197 heartbeating to localhost/127.0.0.1:38835 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-896134436-172.17.0.3-1733375766197 (Datanode Uuid 00dd1f27-ceaa-4272-b9af-de2e158f6264) service to localhost/127.0.0.1:38835 2024-12-05T05:19:23,050 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/data/data1/current/BP-896134436-172.17.0.3-1733375766197 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T05:19:23,050 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/cluster_325d2c52-2715-ad97-5a6c-934718a3c36c/data/data2/current/BP-896134436-172.17.0.3-1733375766197 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T05:19:23,051 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T05:19:23,057 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3f7463d{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T05:19:23,057 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21e93b68{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T05:19:23,057 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T05:19:23,057 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2e580bc0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T05:19:23,057 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1efa708{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/0d95b83c-8c4d-3956-703b-3e0fc956afc0/hadoop.log.dir/,STOPPED} 2024-12-05T05:19:23,070 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-05T05:19:23,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down